/*****************************************************************************
 *  ENTROPY - emerging network to reduce orwellian potency yield
 *
 *  Copyright (C) 2002 Juergen Buchmueller <pullmoll@stop1984.com>
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software Foundation,
 *  Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
 *
 *	$Id: file.c,v 1.39 2006/02/21 16:17:11 pullmoll Exp $
 *****************************************************************************/
#include "file.h"

/* do not log secrets (encryption keys), not even with L_DEBUG or higher */
#define	PARANOID	1

#define	CODEC_UNKNOWN	0
#define	CODEC_FEC_8_4	1

#undef	MIN
#undef	MAX
#define	MIN(a,b) ((a)<(b)?(a):(b))
#define	MAX(a,b) ((a)>(b)?(a):(b))

/**
 * @brief Maximum delay on RNF errors (queue overruns)
 */
#define	MAXDELAY	((int64_t)480 * 1000000 / g_conf->fcpretries)

/**
 * @brief Squeeze a buffer using gzip compress
 *
 * Squeeze a buffer at 'src' of length 'size' bytes using gzip compress.
 * The result is dynamically allocated and has to released by the caller
 * using xfree() after use.
 *
 * @param pbuff pointer to a pointer to buffer to receive compress result
 * @param psize pointer to a size_t to receive the compressed size
 * @param src pointer to the source buffer
 * @param size size of the source buffer
 *
 * @result zero on success, -1 on error (errno = EINVAL for bad parameters)
 */
int file_squeeze(uint8_t **pbuff, size_t *psize, char *src, size_t size)
{
	uint8_t *buff;
	uLong buffsize;
	int e, rc;
	FUN("file_squeeze");

	*pbuff = NULL;
	*psize = 0;
	if (size > 0xfffffff) {
		LOGS(L_FILE,L_ERROR,("buffer is too big (%#x)\n",
			(unsigned)size));
		errno = EINVAL;
		return -1;
	}

	buffsize = size + size / 10 + 16;
	buff = (uint8_t *)xcalloc(12 + buffsize, sizeof(uint8_t));
	rc = compress(&buff[12], &buffsize, (uint8_t *)src, size);
	if (Z_OK != rc) {
		e = errno;
		switch (rc) {
		case Z_STREAM_END:
			LOGS(L_FILE,L_ERROR,("failed to compress() XML %#x bytes string (%s)\n",
				(unsigned)size, "Z_STREAM_END"));
			e = EINVAL;
			break;
		case Z_NEED_DICT:
			LOGS(L_FILE,L_ERROR,("failed to compress() XML %#x bytes string (%s)\n",
				(unsigned)size, "Z_NEED_DICT"));
			e = EINVAL;
			break;
		case Z_ERRNO:
			LOGS(L_FILE,L_ERROR,("failed to compress() XML %#x bytes string (%s)\n",
				(unsigned)size, strerror(errno)));
			break;
		case Z_STREAM_ERROR:
			LOGS(L_FILE,L_ERROR,("failed to compress() XML %#x bytes string (%s)\n",
				(unsigned)size, "Z_STREAM_ERROR"));
			e = EINVAL;
			break;
		case Z_DATA_ERROR:
			LOGS(L_FILE,L_ERROR,("failed to compress() XML %#x bytes string (%s)\n",
				(unsigned)size, "Z_DATA_ERROR"));
			e = EINVAL;
			break;
		case Z_MEM_ERROR:
			LOGS(L_FILE,L_ERROR,("failed to compress() XML %#x bytes string (%s)\n",
				(unsigned)size, "Z_MEM_ERROR"));
			e = EINVAL;
			break;
		case Z_BUF_ERROR:
			LOGS(L_FILE,L_ERROR,("failed to compress() XML %#x bytes string (%s)\n",
				(unsigned)size, "Z_BUF_ERROR"));
			e = EINVAL;
			break;
		case Z_VERSION_ERROR:
			LOGS(L_FILE,L_ERROR,("failed to compress() XML %#x bytes string (%s)\n",
				(unsigned)size, "Z_VERSION_ERROR"));
			e = EINVAL;
			break;
		default:
			LOGS(L_FILE,L_ERROR,("failed to compress() XML %#x bytes string (unknown: %d)\n",
				(unsigned)size, rc));
			e = EINVAL;
			break;
		}
		xfree(buff);
		errno = e;
		return -1;
	}
	LOGS(L_FILE,L_DEBUG,("compress(%#x) -> %#x\n",
		(unsigned)size, (unsigned)buffsize));

	buffsize += 12;

	/* reallocate buffer to required size */
	buff = (uint8_t*)xrealloc(buff, buffsize);

	memcpy(buff, KEY_VERSION, 4);
	*(uint32_t*)&buff[4] = htonl(buffsize);
	*(uint32_t*)&buff[8] = htonl(size);

	*pbuff = buff;
	*psize = buffsize;
	return 0;
}

/**
 * @brief Expand a buffer using gzip uncompress
 *
 * Expand a buffer at 'src' of length 'size' bytes using gzip
 * uncompress(). The required output buffer is allocated and
 * store into the pointer pbuff provided by the caller.
 * The resulting output buffer's size is stored in *psize.
 *
 * @param pbuff pointer to a pointer to buffer to receive uncompress result
 * @param psize pointer to a size_t to receive uncompressed size
 * @param src pointer to the source buffer
 * @param size size of the source buffer
 *
 * @result zero on success, -1 on error (errno = EINVAL for bad parameters)
 */
int file_expand(char **pbuff, size_t *psize, uint8_t *src, size_t size)
{
	uint8_t *buff;
	size_t gzsize, srcsize;
	uLong buffsize;
	int e, rc = 0;
	FUN("file_expand");

	*pbuff = NULL;
	*psize = 0;

	if (0 != memcmp(KEY_VERSION, src, 4)) {
		LOGS(L_FILE,L_ERROR,("This is not a '%s' gzipped buffer (%#x)\n",
			KEY_VERSION, (unsigned)size));
		errno = EINVAL;
		return -1;
	}

	srcsize = ntohl(*(uint32_t*)&src[4]);
	if (srcsize < 12 || srcsize > size + 12) {
		LOGS(L_FILE,L_ERROR,("wrong src size %#x\n",
			(unsigned)srcsize));
		errno = EINVAL;
		return -1;
	}

	buffsize = ntohl(*(uint32_t*)&src[8]);
	if (buffsize < 4 || buffsize > 0xfffffff) {
		LOGS(L_FILE,L_ERROR,("wrong XML size %#x\n",
			(unsigned)(buffsize)));
		errno = EINVAL;
		return -1;
	}

	gzsize = size - 12;

	buff = (uint8_t *)xcalloc(buffsize + 1, sizeof(uint8_t));
	rc = uncompress(buff, &buffsize, &src[12], gzsize);
	if (Z_OK != rc) {
		e = errno;
		switch (rc) {
		case Z_STREAM_END:
			LOGS(L_FILE,L_ERROR,("failed to uncompress() %#x to %#x bytes data (%s)\n",
				(unsigned)gzsize, (unsigned)buffsize, "Z_STREAM_END"));
			e = EINVAL;
			break;
		case Z_NEED_DICT:
			LOGS(L_FILE,L_ERROR,("failed to uncompress() %#x to %#x bytes data (%s)\n",
				(unsigned)gzsize, (unsigned)buffsize, "Z_NEED_DICT"));
			e = EINVAL;
			break;
		case Z_ERRNO:
			LOGS(L_FILE,L_ERROR,("failed to uncompress() %#x to %#x bytes data (%s)\n",
				(unsigned)gzsize, (unsigned)buffsize, strerror(errno)));
			break;
		case Z_STREAM_ERROR:
			LOGS(L_FILE,L_ERROR,("failed to uncompress() %#x to %#x bytes data (%s)\n",
				(unsigned)gzsize, (unsigned)buffsize, "Z_STREAM_ERROR"));
			e = EINVAL;
			break;
		case Z_DATA_ERROR:
			LOGS(L_FILE,L_ERROR,("failed to uncompress() %#x to %#x bytes data (%s)\n",
				(unsigned)gzsize, (unsigned)buffsize, "Z_DATA_ERROR"));
			e = EINVAL;
			break;
		case Z_MEM_ERROR:
			LOGS(L_FILE,L_ERROR,("failed to uncompress() %#x to %#x bytes data (%s)\n",
				(unsigned)gzsize, (unsigned)buffsize, "Z_MEM_ERROR"));
			e = EINVAL;
			break;
		case Z_BUF_ERROR:
			LOGS(L_FILE,L_ERROR,("failed to uncompress() %#x to %#x bytes data (%s)\n",
				(unsigned)gzsize, (unsigned)buffsize, "Z_BUF_ERROR"));
			e = EINVAL;
			break;
		case Z_VERSION_ERROR:
			LOGS(L_FILE,L_ERROR,("failed to uncompress() %#x to %#x bytes data (%s)\n",
				(unsigned)gzsize, (unsigned)buffsize, "Z_VERSION_ERROR"));
			e = EINVAL;
			break;
		default:
			LOGS(L_FILE,L_ERROR,("failed to uncompress() %#x to %#x bytes data (unknown: %d)\n",
				(unsigned)gzsize, (unsigned)buffsize, rc));
			e = EINVAL;
			break;
		}
		xfree(buff);
		errno = e;
		return -1;
	}
	LOGS(L_FILE,L_DEBUG,("uncompress(%#x) -> %#x\n",
		(unsigned)size, (unsigned)buffsize));

	*pbuff = (char *)buff;
	*psize = buffsize;
	return rc;
}

/**
 * @brief Zap (delete immediately) a number of keys from the data store
 *
 * Zap (delete immediately) 'count' chunks listed in 'chunk'.
 *
 * @param chunk pointer to a list of SHA1 digests
 * @param count number of SHA1 entries in the list
 *
 * @result zero on success (always)
 */
static int file_zap_keys(sha1_digest_t *chunk, size_t count)
{
	size_t n;
	FUN("file_zap_keys");

	for (n = 0; n < count; n++) {
		if (0 == store_zap(&chunk[n])) {
			LOGS(L_FILE,L_DEBUG,("store_zap(%s) succeeded\n",
				sha1_hexshort(&chunk[n])));
		} else if (ENOENT != errno) {
			LOGS(L_FILE,L_ERROR,("store_zap(%s) failed (%s)\n",
				sha1_hexshort(&chunk[n]),
				strerror(errno)));
		}
	}
	return 0;
}

/*
 * @brief Delete (soon) a number of keys from the data store
 *
 * Delete (soon) 'count' chunks listed in 'chunk'.
 *
 * @param chunk pointer to a list of SHA1 digests
 * @param count number of SHA1 entries in the list
 *
 * @result zero on success (always)
 */
static int file_del_keys(sha1_digest_t *chunk, size_t count)
{
	size_t n;
	FUN("file_del_keys");

	for (n = 0; n < count; n++) {
		if (0 == store_del(&chunk[n])) {
			LOGS(L_FILE,L_DEBUG,("store_del(%s) succeeded\n",
				sha1_hexshort(&chunk[n])));
		} else if (ENOENT != errno) {
			LOGS(L_FILE,L_ERROR,("store_del(%s) failed (%s)\n",
				sha1_hexshort(&chunk[n]),
				strerror(errno)));
		}
	}
	return 0;
}

/**
 * @brief Return a hops-to-live value for try number 'try'
 *
 * Return the retry HTL value from 'htl' for try number 'try'
 * Scaling range is 1 to htl*try/tries (e.g. htl*4/5).
 * The first try returns the original htl value.
 *
 * @param htl hops-to-live value to scale
 * @param try (re-)try number
 *
 * @result a hops-to-live value in range 2 ... htl
 */
static int retry_htl(int htl, int try)
{
	if (htl <= 0)
		return 0;

	if (try > 1)
		htl = htl * try / g_conf->fcpretries;
	if (htl < 1)
		return 2;
	return htl;
}

/**
 * @brief Try to get and/or reconstruct a fragment of some (unknown) file
 *
 * Try to get and/or reconstruct a fragment. The 13 keys are listed in 's',
 * where the first entry contains the fragment's SHA1, and the following 12
 * the SHA1 of the chunks. If successful, advertize the reconstructed
 * chunks.
 *
 * @param s pointer to an array of 13 SHA1 digests
 * @param htl hops-to-live value to use
 *
 * @result zero on success, -1 on error
 */
int file_get_frag(sha1_digest_t *s, int htl)
{
	sha1_digest_t sha1_check;
	sha1_digest_t *chunk = &s[1];
	uint8_t *buff = NULL;
	uint8_t *code = NULL;
	uint32_t sum;
	int64_t limit;
	int64_t delay;
	size_t n;
	int bit;
	int try;
	int h;
	int reqflags = 0;
	int badflags = 0;
	int advflags = 0;
	int e;
	int rc = 0;
	FUN("file_get_frag");

	/* allocate a buffer for the chunks */
	code = (uint8_t *)xcalloc(12, CHUNKSIZE);

	reqflags = 0;
	sum = bitsum(reqflags);
	for (try = 1; try <= g_conf->fcpretries; try++) {
		h = retry_htl(htl, try);
		if (try > 1) {
			limit = MAX(g_peer->in.available, g_conf->bwlimit_in/32);
			delay = (int64_t)1000000 * try * (12-sum) *
				MAX_MSGSIZE / limit / g_conf->fcpretries;
			/* No delays longer than MAXDELAY microseconds */
			delay = MIN(delay, MAXDELAY);
			LOGS(L_FILE,L_MINOR,("%s try #%d %d/12 (%s) HTL:%d %s\n",
				sha1_hexshort(s), try, sum,
				flags_str(reqflags), h, usec_str(delay)));
			osd_usleep(delay);
		}
		/* request the remaining of the 12 chunks */
		rc = peer_req_wait(chunk, 12, &reqflags, h, code);
		sum = bitsum(reqflags);
		if (0 == rc)
			break;
		if (EINVAL == errno)
			goto bailout;
		if (sum >= 8) {
			e = errno;
			LOGS(L_FILE,L_MINOR,("%d/12 chunks but rc was %d (%s)\n",
				sum, rc, strerror(errno)));
			rc = 0;
			errno = e;
			break;
		}
		if (0 == htl)
			break;
	}

	if (0 != rc) {
		e = errno;
		LOGS(L_FILE,L_MINOR,("per_req_wait(%s) call failed (%s)\n",
			sha1_hexshort(s), flags_str(reqflags)));
		errno = e;
		goto bailout;
	}

	/* check the chunks and zero out the ones that don't match the SHA1 */
	for (n = 0; n < 12; n++) {
		uint8_t *data = &code[n * CHUNKSIZE];

		bit = 1 << n;
		/* skip chunks that were not retrieved on request */
		if (0 == (reqflags & bit))
			continue;

		sha1(data, CHUNKSIZE, &sha1_check);

		if (0 != memcmp(&sha1_check, &chunk[n], SHA1SIZE)) {
			LOGS(L_FILE,L_ERROR,("fragment chunk #%#x bad\n",
				(unsigned)n));
			/* erase this chunk */
			memset(data, 0, CHUNKSIZE);
			badflags |= bit;
			reqflags &= ~bit;
		}
	}

	buff = (uint8_t *)xmalloc(FRAGSIZE);

	rc = fec84_decode(buff, code, FRAGSIZE, reqflags);
	if (0 != rc) {
		LOGS(L_FILE,L_MINOR,("fec84_decode() failed (%s)\n",
			strerror(errno)));
		goto bailout;
	}
	LOGS(L_FILE,L_MINOR,("fec84_decode() succeeded (%s)\n",
		flags_str(reqflags)));

	sha1(buff, FRAGSIZE, &sha1_check);
	if (0 != memcmp(s, &sha1_check, SHA1SIZE)) {
		LOGS(L_FILE,L_MINOR,("SHA1 mismatch %s:%s\n",
			sha1_hexshort(s),
			sha1_hexshort(&sha1_check)));
		errno = EINVAL;
		rc = -1;
		goto bailout;
	}
	LOGS(L_FILE,L_MINOR,("SHA1 for fragment matches (%s)\n",
		sha1_hexshort(s)));

	/* Now that we know the fragment is valid, forward the request */
	rc = peer_req_fec(s, 2, htl, NULL, 0);
	if (0 == rc) {
		LOGS(L_FILE,L_MINOR,("peer_req_fec() succeeded\n"));
	} else {
		LOGS(L_FILE,L_MINOR,("peer_req_fec() failed (%s)\n",
			strerror(errno)));
	}

	if (0x000 != badflags) {
		LOGS(L_FILE,L_MINOR,("zapping bad chunks (%s)\n",
			flags_str(badflags)));
		/* zap the chunks that didn't match the SHA1 */
		for (n = 0; n < 12; n++) {
			bit = 1 << n;

			/* skip chunks that were okay */
			if (0 == (badflags & bit))
				continue;

			rc = store_zap(&chunk[n]);
			if (0 == rc) {
				LOGS(L_FILE,L_MINOR,("store_zap(%s) succeeded\n",
					sha1_hexshort(&chunk[n])));
			} else if (ENOENT != errno) {
				LOGS(L_FILE,L_ERROR,("store_zap(%s) failed (%s)\n",
					sha1_hexshort(&chunk[n]), strerror(errno)));
			}
		}
	}

	/*
	 * Encode any missing check chunks, except the ones that are
	 * already non-zero in the reqflags.
	 */
	rc = fec84_encode(code, buff, FRAGSIZE, reqflags);
	if (0 != rc) {
		LOGS(L_FILE,L_MINOR,("fec84_encode() failed (%s)\n",
			strerror(errno)));
		goto bailout;
	}

	/* first set all flags, so none would be advertized */
	advflags = 0xfff;

	/* check each chunk's SHA1 and store its data */
	for (n = 0; n < 12; n++) {
		uint8_t *data = &code[n * CHUNKSIZE];

		bit = 1 << n;

		sha1(data, CHUNKSIZE, &sha1_check);

		rc = memcmp(&sha1_check, &chunk[n], SHA1SIZE);
		if (0 != rc) {
			/* someone trying to make us encode wrong chunks? */
			LOGS(L_FILE,L_NORMAL,("fragment chunk #%#x bad\n",
				(unsigned)n));
			errno = EINVAL;
			goto bailout;
		} else if (0 == (reqflags & bit)) {
			rc = store_put(&chunk[n], data);
			if (0 != rc) {
				LOGS(L_FILE,L_ERROR,("chunk #%#x store_put(%s) failed (%s)\n",
					(unsigned)n, sha1_hexshort(&chunk[n]),
					strerror(errno)));
				goto bailout;
			}
			LOGS(L_FILE,L_MINOR,("chunk #%#x store_put(%s) succeeded\n",
				(unsigned)n, sha1_hexshort(&chunk[n])));
			reqflags |= bit;
			advflags &= ~bit;
		}
	}

	/* If none were reconstructed, select up to three at random */
	if (advflags == 0xfff) {
		uint32_t bits;
		/* advertize up to 3 chunks */
		if (0 != rnd_get(&bits, sizeof(bits))) {
			const char *errmsg = strerror(errno);
			LOGS(L_FILE,L_ERROR,("rnd_get() failed (%s)\n",
				errmsg));
			die(1, "rnd_get() failed (%s)", errmsg);
		}
		/* set bits of advflags from 3 nibbles of 'bits' */
		advflags = 1 << ((bits >> 0) % 12);
		advflags |= 1 << ((bits >> 4) % 12);
		advflags |= 1 << ((bits  >> 8) % 12);
		LOGS(L_FILE,L_MINOR,("ADV randomly selected %s\n",
			flags_str(advflags)));
		/* bit = 0 means "still need to advertize", so flip the mask */
		advflags ^= 0xfff;
	}
	/* advertize the selected chunks of this fragment */
	rc = peer_adv_wait(chunk, 12, &advflags, htl);
	if (0 == rc) {
		LOGS(L_FILE,L_MINOR,("peer_adv_wait(%s) succeeded\n",
			flags_str(advflags)));
	} else {
		LOGS(L_FILE,L_MINOR,("peer_adv_wait() failed (%s)\n",
			strerror(errno)));
	}

bailout:
	xfree(code);
	xfree(buff);

	return rc;
}

/**
 * @brief Put a (small) file into the key's XML data tage
 *
 * Put a (small) file into a <data ...> tag inside the key's XML code.
 * The compressed XML data must fit into CHUNKSIZE to make a short key,
 * otherwise the file has to be encoded using file_put() chunking.
 *
 * @param key pointer to a chkey_t to receive the resulting key
 * @param tfdata pointer to a tempfile_t containing the data
 * @param tfmeta pointer to a tempfile_t containing the meta data
 * @param htl hops-to-live for the insert
 * @param collision pointer to an int that is set to 1 if the key was known
 * @param user user argument for the callback
 * @param cb callback to progress info function
 * @param flags flags from enum with FILE_XXX constants
 *
 * @result zero on success, -1 and errno = ENOSPC if file too big, or other errno
 */
int file_put_direct(chkey_t *key, tempfile_t *tfdata, tempfile_t *tfmeta,
	int htl, int *collision, void *user, file_callback cb, int flags)
{
	size_t datasize = 0, metasize = 0;
	size_t i, offs, guess = 0, done = 0, gzsize = 0, xmlsize = 0;
	char *xml = NULL, *dst = NULL;
	char *dst_key = NULL;
	uint8_t *gzbuff = NULL;
	sha1_state_t ss;
	ek5_state_t es;
	chkey_t key_content;
	int64_t limit, delay;
	int insflags, try, h;
	int e, rc = 0;
	FUN("file_put_direct");

	if (NULL != tfdata) {
		if (0 == strlen(tfdata->name)) {
			/* no data */
			datasize = 0;
		} else if (0 != (rc = temp_open(tfdata))) {
			LOGS(L_FILE,L_DEBUG,("temp_open('%s') call failed (%s)\n",
				tfdata->name ? tfdata->name : "[null]", strerror(errno)));
			goto bailout;
		} else {
			datasize = tfdata->size;
		}
	}

	if (NULL != tfmeta) {
		if (0 == strlen(tfmeta->name)) {
			/* no meta data */
			metasize = 0;
		} else if (0 != (rc = temp_open(tfmeta))) {
			LOGS(L_FILE,L_DEBUG,("temp_open('%s') call failed (%s)\n",
				tfmeta->name ? tfmeta->name : "[null]", strerror(errno)));
			metasize = 0;
		} else {
			metasize = tfmeta->size;
		}
	}

	/* rough estimation of the required meta data buffer */
	guess = 4096 + 2 * datasize + 2 * metasize;
	xml = xcalloc(guess, sizeof(char));

	dst = xml;
	dst += pm_spprintf(dst,xml,guess,
		"<?xml version=\"1.0\" standalone='yes'?>\n");
	dst += pm_spprintf(dst,xml,guess,
		"<content size='%x' key='", datasize);
	dst_key = dst;
	for (i = 0; i < KEY1SIZE + 1 + KEY2SIZE; i++)
		dst += pm_spprintf(dst,xml,guess, "x");
	dst += pm_spprintf(dst,xml,guess,"'>\n");

	sha1_init(&ss);
	ek5_init(&es);
	if (datasize > 0) {
		uint8_t *buff;
		dst += pm_spprintf(dst,xml,guess,"\t<data size='%x'>", datasize);

		buff = (uint8_t *)xmalloc(datasize);
		if (0 != (rc = temp_read(tfdata, buff, datasize, &done))) {
			LOGS(L_FILE,L_ERROR,("temp_read() from '%s' call failed (%s)\n",
				tfdata->name, strerror(errno)));
			xfree(buff);
			goto bailout;
		}
		if (datasize != done) {
			LOGS(L_FILE,L_ERROR,("temp_read() unexpected size %#x (%#x)\n",
				(unsigned)done, (unsigned)datasize));
			xfree(buff);
			errno = EINVAL;
			rc = -1;
			goto bailout;
		}
		/* append the data to the SHA1 and EK5 hashes */
		sha1_append(&ss, buff, done);
		ek5_append(&es, buff, done);

		for (i = 0; i < done; i++) {
			dst += pm_spprintf(dst,xml,guess,"%02x",
				buff[i]);
		}
		dst += pm_spprintf(dst,xml,guess,"</data>\n");
		xfree(buff);
	}

	if (metasize > 0) {
		uint8_t *buff;
		dst += pm_spprintf(dst,xml,guess,"\t<metadata size='%x'>",
			metasize);
		buff = (uint8_t *)xmalloc(metasize);
		if (0 != (rc = temp_read(tfmeta, buff, metasize, &done))) {
			LOGS(L_FILE,L_ERROR,("temp_read() from '%s' call failed (%s)\n",
				tfmeta->name, strerror(errno)));
			xfree(buff);
			rc = -1;
			goto bailout;
		}
		if (metasize != done) {
			LOGS(L_FILE,L_ERROR,("temp_read() unexpected size %#x (%#x)\n",
				(unsigned)done, (unsigned)metasize));
			xfree(buff);
			errno = EINVAL;
			rc = -1;
			goto bailout;
		}
		/* append the meta data to the EK5 hash */
		ek5_append(&es, buff, done);
		/* encode meta data as a string of hex digits */
		for (offs = 0; offs < done; offs++) {
			dst += pm_spprintf(dst,xml,guess,"%02x",
				buff[offs]);
		}
		dst += pm_spprintf(dst,xml,guess,"</metadata>\n");
		xfree(buff);
	}
	dst += pm_spprintf(dst,xml,guess,"</content>\n");

	sha1_finish(&ss, &key_content.sha1);
	ek5_finish(&es, &key_content.ek5);

	if (0 == (flags & FILE_INT)) {
		key_content.type[0] = MSB(K_CHK);
		key_content.type[1] = LSB(K_CHK);
		/* encrypt with the content EK5 */
		key->ek5 = key_content.ek5;
	} else {
		key_content.type[0] = MSB(K_BIT);
		key_content.type[1] = LSB(K_BIT);
		/* store the "parent's" EK5 hash */
		key_content.ek5 = key->ek5;
	}
	key_content.log2size = log2size(datasize);

#if	PARANOID == 0
	LOGS(L_FILE,L_DEBUG,("data SHA1 %s, meta EK5 %s\n",
		sha1_hexshort(&key_content.sha1),
		ek5_hexshort(&key_content.ek5)));
#endif
	memcpy(dst_key, key_str(&key_content), KEY1SIZE + 1 + KEY2SIZE);

	xmlsize = (size_t)(dst - xml);

	rc = file_squeeze(&gzbuff, &gzsize, xml, xmlsize);
	if (0 != rc) {
		LOGS(L_FILE,L_DEBUGX,("file_squeeze() call failed (%s)\n",
			strerror(errno)));
		goto bailout;
	}

	if (gzsize >= CHUNKSIZE) {
		errno = ENOSPC;
		rc = -1;
		goto bailout;
	}
	gzbuff = xrealloc(gzbuff, CHUNKSIZE);
	memset(&gzbuff[gzsize], 0, CHUNKSIZE - gzsize);

	LOGS(L_FILE,L_DEBUG,("data:%#x meta:%#x xml:%#x gz:%#x\n",
		(unsigned)datasize, (unsigned)metasize,
		(unsigned)xmlsize, (unsigned)gzsize));

#if	PARANOID == 0
	LOGS(L_FILE,L_DEBUG,("encryption key: %s\n", ek5_hexshort(&key->ek5)));
#endif

	key_crypt(key->ek5.digest, gzbuff, gzsize, KEY_CRYPT_GZ);

#if	PARANOID == 0
	LOGS(L_FILE,L_DEBUG,("encrypted data:\n%s\n", hexdump(buff, gzsize)));
#endif

	sha1(gzbuff, gzsize, &key->sha1);

	if (0 == (flags & FILE_INT)) {
		key->type[0] = MSB(K_CHK);
		key->type[1] = LSB(K_CHK);
	} else {
		key->type[0] = MSB(K_BIT);
		key->type[1] = LSB(K_BIT);
	}
	key->log2size = log2size(datasize);

	if (NULL != cb) {
		file_info_t info;
		memset(&info, 0, sizeof(info));
		info.type = INFO_TYPE_PROGRESS;
		info.internal = flags & FILE_INT;
		info.hash = key->sha1;
		info.offs = datasize;
		info.size = datasize;
		if (0 != (rc = (*cb)(user, &info))) {
			LOGS(L_FILE,L_ERROR,("callback failed (%s)\n",
				strerror(errno)));
			goto bailout;
		}
	}

	if (htl >= 0) {
		if (0 == (rc = store_get(&key->sha1, NULL))) {
			LOGS(L_FILE,L_DEBUG,("key collision %s\n",
				key_short(key)));
			*collision = 1;
		}
		if (0 != (rc = store_put(&key->sha1, gzbuff))) {
			LOGS(L_FILE,L_ERROR,("store_put() call failed (%s)\n",
				strerror(errno)));
			goto bailout;
		}
	}

	LOGS(L_FILE,L_DEBUG,("short key %s,%s\n",
		sha1_hexshort(&key->sha1),
		ek5_hexshort(&key->ek5)));

	if (htl > 0) {
		insflags = 0;
		for (try = 1; try <= g_conf->fcpretries; try++) {
			h = retry_htl(htl, try);
			if (try > 1) {
				limit = MAX(g_peer->out.available, g_conf->bwlimit_out / 32);
				delay = (int64_t)1000000 * try * (MIN_MSGSIZE+gzsize) /
					limit / g_conf->fcpretries;
				/* No delays longer than MAXDELAY microseconds */
				delay = MIN(delay, MAXDELAY);
				LOGS(L_FILE,L_MINOR,("%s try #%d size:%#x HTL:%d %s\n",
					sha1_hexshort(&key->sha1), try, (unsigned)gzsize,
					h, usec_str(delay)));
				osd_usleep(delay);
			}
			if (0 == *collision) {
				/* insert the key to some peers if possible */
				rc = peer_ins_wait(&key->sha1, 1, &insflags, h, gzbuff);
			} else {
				/* just advertize the key to some peers if possible */
				rc = peer_adv_wait(&key->sha1, 1, &insflags, h);
			}
			if (0 == rc)
				break;
			if (EINVAL == errno)
				goto bailout;
		}
		if (0 != rc) {
			LOGS(L_FILE,L_ERROR,("peer_adv/ins_wait() call failed (%s)\n",
				strerror(errno)));
		}
	}

	if (0 == rc && 0 != (flags & FILE_DEL)) {
		file_del_keys(&key->sha1, 1);
	}

bailout:
	e = errno;	/* how do I keep errno over some function calls? not? */
	temp_close(tfdata, 0);
	temp_close(tfmeta, 0);
	xfree(xml);
	xfree(gzbuff);
	errno = e;
	return rc;
}

/**
 * @brief Put a file into the data store and insert/advertize it
 *
 * Encode a file into fragments of FRAGSIZE bytes size, and store each
 * fragment as set of code chunks of CHUNKSIZE bytes size. A fragment
 * is split into bits and four check bits are added per byte (octet),
 * so this function produces a 50% redundancy for the net data, plus
 * the XML document's key describing the list of fragments and chunks.
 *
 * If a file is "rather short", i.e. < 3*CHUNKSIZE, a first try is
 * made to insert it as data contained in the XML itself, using the
 * file_put_direct() call.
 *
 * @param key pointer to a chkey_t to receive the resulting key
 * @param tfdata pointer to a tempfile_t containing the data
 * @param tfmeta pointer to a tempfile_t containing the meta data
 * @param htl hops-to-live for the insert
 * @param collision pointer to an int that is set to 1 if the key was known
 * @param user user argument for the callback
 * @param cb callback to progress info function
 * @param flags flags from enum with FILE_XXX constants
 *
 * @result zero on success, -1 and errno set on error
 */
int file_put(chkey_t *key, tempfile_t *tfdata, tempfile_t *tfmeta,
	int htl, int *collision, void *user, file_callback cb, int flags)
{
	size_t datasize = 0;
	size_t metasize = 0;
	size_t guess = 0;
	size_t gzsize = 0;
	size_t xmlsize = 0;
	uint8_t *buff = NULL;
	uint8_t *code = NULL;
	uint8_t *gzbuff = NULL;
	ek5_state_t es;
	sha1_state_t ss;
	sha1_digest_t sha1_fec[12+1];
	sha1_digest_t sha1_chunk[12];
	char *xml = NULL, *dst = NULL;
	char hexbuf[10];
	char *dst_hash;
	char *dst_frags;
	size_t i, n, offs, size, fragsize;
	size_t fragnum = 0, fragcnt = 0;
	chkey_t key_frag;
	chkey_t key_content;
	int64_t limit, delay;
	int advflags = 0;
	int insflags = 0;
	int bit, try, h;
	int e;
	int rc = 0;
	FUN("file_put");

	*collision = 0;

	buff = (uint8_t *)xmalloc(FRAGSIZE);
	code = (uint8_t *)xmalloc(12 * CHUNKSIZE);

	if (NULL != tfdata) {
		if (0 == strlen(tfdata->name)) {
			/* no data */
			datasize = 0;
		} else if (0 != (rc = temp_open(tfdata))) {
			LOGS(L_FILE,L_ERROR,("temp_open('%s') call failed (%s)\n",
				tfdata->name, strerror(errno)));
			datasize = 0;
		} else {
			datasize = tfdata->size;

		}
	}

	if (NULL != tfmeta) {
		if (0 == strlen(tfmeta->name)) {
			/* no meta data */
			metasize = 0;
		} else if (0 != (rc = temp_open(tfmeta))) {
			LOGS(L_FILE,L_ERROR,("temp_open('%s') call failed (%s)\n",
				tfmeta->name, strerror(errno)));
			metasize = 0;
		} else {
			metasize = tfmeta->size;
		}
	}

	/* Try if the data and meta data fits into a single chunk */
	size = datasize + metasize;
	if (size > 0 && size < 3*CHUNKSIZE) {
		chkey_t short_key = *key;
		temp_close(tfdata, 0);
		temp_close(tfmeta, 0);
		rc = file_put_direct(&short_key, tfdata, tfmeta,
			htl, collision, user, cb, flags);
		/* The document compressed well enough _and_ insert succeeded */
		if (0 == rc) {
			*key = short_key;
			goto bailout;
		}
		/*
		 * Every other error is just this error, and does not mean:
		 * make this document a fragmented XML
		 */
		if (ENOSPC != errno) {
			*key = short_key;
			goto bailout;
		}
		temp_open(tfdata);
		temp_open(tfmeta);
	}

	sha1_init(&ss);
	ek5_init(&es);

	/* rough estimation of the required meta data buffer */
	fragcnt = (datasize + FRAGSIZE - 1) / FRAGSIZE;
	guess = 4096 + fragcnt * 12 * 128 + 2 * metasize;
	xml = xcalloc(guess, sizeof(char));

	dst = xml;
	dst += pm_spprintf(dst,xml,guess,
		"<?xml version=\"1.0\" standalone='yes'?>\n");
	dst += pm_spprintf(dst,xml,guess,
		"<content size='%x' key='", datasize);
	dst_hash = dst;
	for (i = 0; i < KEY1SIZE + 1 + KEY2SIZE; i++)
		dst += pm_spprintf(dst,xml,guess, "x");
	dst += pm_spprintf(dst,xml,guess,"' fragments='");
	dst_frags = dst;
	dst += pm_spprintf(dst,xml,guess,"xxxxxxxx'>\n");

	if (datasize > metasize) {
		for (fragnum = 0; /* */; fragnum++) {

			/* read the next fragment */	
			if (0 != (rc = temp_read(tfdata, buff, FRAGSIZE, &fragsize))) {
				LOGS(L_FILE,L_ERROR,("temp_read() from '%s' call failed (%s)\n",
					tfdata->name, strerror(errno)));
				goto bailout;
			}
			if (fragsize == 0) {
				/* end of file */
				break;
			}
			if (fragsize < FRAGSIZE) {
				memset(&buff[fragsize], 0, FRAGSIZE - fragsize);
			}

			/* EK5 of the unencrypted fragment data */
			ek5(buff, FRAGSIZE, &key_frag.ek5);
			/* encrypt fragment with the EK5 hash of the unecrypted data */
			key_crypt(key_frag.ek5.digest, buff, fragsize, 0);
			/* SHA1 of the encrypted fragment */
			sha1(buff, FRAGSIZE, &key_frag.sha1);

			key_frag.type[0] = MSB(K_BIT);
			key_frag.type[1] = LSB(K_BIT);
			key_frag.log2size = log2size(fragsize);

			LOGS(L_FILE,L_MINOR,("frag #%#x/%#x %#x/%#x SHA1 %s\n",
				(unsigned)(fragnum + 1),
				(unsigned)fragcnt,
				(unsigned)fragsize,
				(unsigned)FRAGSIZE,
				sha1_hexshort(&key_frag.sha1)));
	
			/* append encrypted fragment to the SHA1 of the entire data */
			sha1_append(&ss, buff, FRAGSIZE);
			ek5_append(&es, buff, FRAGSIZE);

			if (NULL != cb) {
				file_info_t info;
				memset(&info, 0, sizeof(info));
				info.type = INFO_TYPE_PROGRESS;
				info.internal = flags & FILE_INT;
				info.hash = key_frag.sha1;
				info.offs = fragnum * FRAGSIZE;
				info.size = datasize + metasize;
				if (0 != (rc = (*cb)(user, &info))) {
					LOGS(L_FILE,L_ERROR,("callback failed (%s)\n",
						strerror(errno)));
					goto bailout;
				}
			}

			/* FEC84 encode the encrypted fragment (0 = all bits) */
			rc = fec84_encode(code, buff, FRAGSIZE, 0);
			if (0 != rc) {
				LOGS(L_FILE,L_ERROR,("fec84 encode frag #%#x/%#x returned %d\n",
					(unsigned)(fragnum + 1),
					(unsigned)fragcnt,
					rc));
				goto bailout;
			}

			dst += pm_spprintf(dst,xml,guess,"\t<fragment number='%x'",
				fragnum);
			dst += pm_spprintf(dst,xml,guess," mode='%s'",
				"fec84");
			dst += pm_spprintf(dst,xml,guess," size='%x'",
				(unsigned)fragsize);
			dst += pm_spprintf(dst,xml,guess," key='%s'>\n",
				key_str(&key_frag));
	
			/* find SHA1 hashes of the encrypted and FEC84 encoded chunks */
			for (n = 0; n < 12; n++) {
				uint8_t *data = &code[n * CHUNKSIZE];

				/* get SHA1 of this bits chunk */
				sha1(data, CHUNKSIZE, &sha1_chunk[n]);

				LOGS(L_FILE,L_DEBUG,("%s #%#x hash %s\n",
					"fec84", (unsigned)n,
					sha1_hexshort(&sha1_chunk[n])));
				dst += pm_spprintf(dst,xml,guess,"\t\t<chunk");
				dst += pm_spprintf(dst,xml,guess," number='%x'", (unsigned)n);
				dst += pm_spprintf(dst,xml,guess," hash='%s' />\n",
					sha1_hexstr(&sha1_chunk[n]));
			}
			dst += pm_spprintf(dst,xml,guess,"\t</fragment>\n");

			/* store the chunks only if HTL is greater than or equal to zero */
			advflags = 0;
			if (0 == (flags & FILE_META) && htl >= 0) {
				for (n = 0; n < 12; n++) {
					uint8_t *data = &code[n * CHUNKSIZE];
					bit = 1 << n;

					if (0 == store_get(&sha1_chunk[n], NULL)) {
						/* it isn't new, so just advertize this chunk */
						advflags |= bit;
					}
					if (0 != (rc = store_put(&sha1_chunk[n], data))) {
						LOGS(L_FILE,L_ERROR,("store_put(%s) failed (%s)\n",
							sha1_hexshort(&sha1_chunk[n]),
							strerror(errno)));
						goto bailout;
					}
				}
			}

			/* only if HTL is greater than zero insert/advertize the chunks */
			if (0 == (flags & FILE_META) && htl > 0) {

				/*
				 * Insert new chunks to some peers, and
				 * just advertize previously known chunks
				 */
				insflags = advflags;
				advflags = advflags ^ 0xfff;
				for (try = 1; try <= g_conf->fcpretries; try++) {
					h = retry_htl(htl, try);
					if (try > 1) {
						/* linearly increasing delays between ??? and MAXDELAY */
						delay = (int64_t)MAXDELAY * try / g_conf->fcpretries;
						LOGS(L_FILE,L_MINOR,("try #%d sleep: %s\n",
							try, usec_str(delay)));
						osd_usleep(delay);
					}
					/* first try to insert the new chunks */
					rc = 0;
					if (bitsum(insflags) < 12)
						rc |= peer_ins_wait(sha1_chunk, 12, &insflags, h, code);
					/* second advertize previously known chunks */
					if (bitsum(advflags) < 12)
						rc |= peer_adv_wait(sha1_chunk, 12, &advflags, h);
					if (0 == rc)
						break;
					if (EINVAL == errno)
						goto bailout;
					LOGS(L_FILE,L_MINOR,("%s try #%d ins:%d(%s)/adv:%d(%s)/12 HTL:%d\n",
						sha1_hexshort(sha1_fec), try,
						bitsum(insflags), flags_str(insflags),
						bitsum(advflags), flags_str(advflags), h));
				}

				if (0 != rc) {
					LOGS(L_FILE,L_MINOR,("insert/advertize frag #%#x/%#x failed\n",
						(unsigned)(fragnum + 1),
						(unsigned)fragcnt));
					goto bailout;
				}

				/* prepare the 13 peer_req_fec() hashes */
				memcpy(&sha1_fec[0], &key_frag.sha1, SHA1SIZE);
				memcpy(&sha1_fec[1], &sha1_chunk[0], 12 * SHA1SIZE);
				rc = peer_req_fec(sha1_fec, 2, htl, NULL, 0);
				if (0 == rc) {
					LOGS(L_FILE,L_MINOR,("peer_req_fec() succeeded\n"));
				} else {
					LOGS(L_FILE,L_MINOR,("peer_req_fec() failed (%s)\n",
						strerror(errno)));
					/* reset result code, though (no error) */
					rc = 0;
				}
			}

			/* shall the file be deleted? */
			if (0 != (flags & FILE_DEL)) {
				file_del_keys(sha1_chunk, 12);
			}
		}
	}

	if (metasize > 0) {
		dst += pm_spprintf(dst,xml,guess,"\t<metadata size='%x'>",
			metasize);
		for (;;) {
			size_t done;
			if (0 != (rc = temp_read(tfmeta, buff, FRAGSIZE, &done))) {
				LOGS(L_FILE,L_ERROR,("temp_read() from '%s' call failed (%s)\n",
					tfmeta->name, strerror(errno)));
				rc = -1;
				goto bailout;
			}
			if (0 == done) {
				break;
			}
			/* append this chunk of meta data to the EK5 hash */
			ek5_append(&es, buff, done);
			/* encode meta data as a string of hex digits */
			for (offs = 0; offs < done; offs++) {
				dst += pm_spprintf(dst,xml,guess,"%02x", buff[offs]);
			}
		}
		dst += pm_spprintf(dst,xml,guess,"</metadata>\n");
	}
	dst += pm_spprintf(dst,xml,guess,"</content>\n");

	/* finish the SHA1/EK5 hashes of the encrypted data (and meta data) */
	sha1_finish(&ss, &key_content.sha1);
	ek5_finish(&es, &key_content.ek5);

	if (0 == (flags & FILE_INT)) {
		key_content.type[0] = MSB(K_CHK);
		key_content.type[1] = LSB(K_CHK);
		/* Encrypt with the content EK5 hash */
		key->ek5 = key_content.ek5;
	} else {
		key_content.type[0] = MSB(K_BIT);
		key_content.type[1] = LSB(K_BIT);
		/* store the "parent's" EK5 hash */
		key_content.ek5 = key->ek5;
	}
	key_content.log2size = log2size(datasize);

	memcpy(dst_hash, key_str(&key_content), KEY1SIZE + 1 + KEY2SIZE);

	pm_snprintf(hexbuf, sizeof(hexbuf), "%08x", fragnum);
	memcpy(dst_frags, hexbuf, 8);

	xmlsize = (size_t)(dst - xml);

	temp_close(tfdata, 0);
	temp_close(tfmeta, 0);

	LOGS(L_FILE,L_DEBUG,("key %s,%s with %#x fragments\n" \
		"real size %u, guess %u\n%s\n",
		sha1_hexshort(&key_content.sha1),
		ek5_hexshort(&key_content.ek5),
		(unsigned)fragnum,
		(unsigned)xmlsize,
		(unsigned)guess,
		xml));

	rc = file_squeeze(&gzbuff, &gzsize, xml, xmlsize);
	if (0 != rc) {
		LOGS(L_FILE,L_DEBUGX,("file_squeeze() call failed (%s)\n",
			strerror(errno)));
		goto bailout;
	}
	xfree(xml);

	LOGS(L_FILE,L_DEBUG,("data:%#x meta:%#x xml:%#x gz:%#x\n",
		(unsigned)datasize, (unsigned)metasize,
		(unsigned)xmlsize, (unsigned)gzsize));

	if (gzsize < CHUNKSIZE) {

		LOGS(L_FILE,L_DEBUG,("XML compressed size %#x; direct\n",
			(unsigned)gzsize));
		gzbuff = (uint8_t *)xrealloc(gzbuff, CHUNKSIZE);
		memset(&gzbuff[gzsize], 0, CHUNKSIZE - gzsize);

		LOGS(L_FILE,L_DEBUG,("encryption key: %s\n", ek5_hexshort(&key->ek5)));
		key_crypt(key->ek5.digest, gzbuff, gzsize, KEY_CRYPT_GZ);
		sha1(gzbuff, gzsize, &key->sha1);

		/* only if HTL is greater or equal zero store the key */
		if (htl >= 0) {
			if (0 == (rc = store_get(&key->sha1, NULL))) {
				LOGS(L_FILE,L_DEBUG,("key collision %s\n",
					key_short(key)));
				*collision = 1;
			}
			if (0 != (rc = store_put(&key->sha1, gzbuff))) {
				LOGS(L_FILE,L_ERROR,("store_put() call failed (%s)\n",
					strerror(errno)));
				goto bailout;
			}
		}

		LOGS(L_FILE,L_DEBUG,("direct key %s,%s\n",
			sha1_hexshort(&key->sha1),
			ek5_hexshort(&key->ek5)));

		if (htl > 0) {
			insflags = 0;
			for (try = 1; try <= g_conf->fcpretries; try++) {
				h = retry_htl(htl, try);
				if (try > 1) {
					limit = MAX(g_peer->out.available, g_conf->bwlimit_out / 32);
					delay = (int64_t)1000000 * try *
						(MIN_MSGSIZE + gzsize) / limit /
						g_conf->fcpretries;
					/* No delays longer than MAXDELAY microseconds */
					delay = MIN(delay, MAXDELAY);
					LOGS(L_FILE,L_MINOR,("%s try #%d size:%#x HTL:%d %s\n",
						sha1_hexshort(&key->sha1), try,
						(unsigned)gzsize, h, usec_str(delay)));
					osd_usleep(delay);
				}
				if (0 == *collision) {
					/* new key: insert the key to some peers if possible */
					rc = peer_ins_wait(&key->sha1, 1, &insflags, h, gzbuff);
				} else {
					/* known key: advertize the key to some peers if possible */
					rc = peer_adv_wait(&key->sha1, 1, &insflags, h);
				}
				if (0 == rc)
					break;
				if (EINVAL == errno)
					goto bailout;
			}
			if (0 != rc) {
				LOGS(L_FILE,L_MINOR,("inserting key %s failed\n",
					sha1_hexshort(&key->sha1)));
				goto bailout;
			}
		}

	} else {
		tempfile_t tfxml;

		LOGS(L_FILE,L_DEBUG,("XML compressed size %#x; indirect\n",
			(unsigned)gzsize));

		if (0 != (rc = temp_binary(&tfxml, "xml", gzsize))) {
			LOGS(L_FILE,L_ERROR,("temp_binary('%s') call failed (%s)\n",
				"xml", strerror(errno)));
			goto bailout;
		}
		if (0 != (rc = temp_write(&tfxml, gzbuff, gzsize))) {
			LOGS(L_FILE,L_ERROR,("temp_write(%u) call failed (%s)\n",
				(unsigned)gzsize, strerror(errno)));
			goto bailout;
		}
		temp_close(&tfxml, 0);
		xfree(gzbuff);

		/* save some space by already deallocating the local buffers */
		xfree(buff);
		xfree(code);

		rc = file_put(key, &tfxml, NULL, htl, collision, user, cb,
			(flags & ~FILE_META) | FILE_INT);

		/* free the temporary file again */
		temp_close(&tfxml, 1);

		LOGS(L_FILE,L_DEBUG,("indirect key %s,%s (rc: %d, collision:%d)\n",
			sha1_hexshort(&key->sha1),
			ek5_hexshort(&key->ek5),
			rc, *collision));
	}

	e = errno;
	if (0 != (flags & FILE_DEL)) {
		file_del_keys(&key->sha1, 1);
	}
	errno = e;

bailout:
	e = errno;	/* how do I keep errno over some function calls? not? */
	if (0 == (flags & FILE_INT)) {
		key->type[0] = MSB(K_CHK);
		key->type[1] = LSB(K_CHK);
	} else {
		key->type[0] = MSB(K_BIT);
		key->type[1] = LSB(K_BIT);
	}
	key->log2size = log2size(datasize);
	temp_close(tfdata, 0);
	temp_close(tfmeta, 0);
	xfree(buff);
	xfree(code);
	xfree(xml);
	xfree(gzbuff);
	if (0 == rc && NULL != cb) {
		file_info_t info;
		memset(&info, 0, sizeof(info));
		info.type = INFO_TYPE_COMPLETED;
		info.internal = flags & FILE_INT;
		info.hash = key->sha1;
		info.offs = datasize + metasize;
		info.size = datasize + metasize;
		if (0 != (rc = (*cb)(user, &info))) {
			LOGS(L_FILE,L_ERROR,("callback failed (%s)\n",
				strerror(errno)));
		}
	}
	errno = e;
	return rc;
}

/**
 * @brief Insert a redirection to a content hash key
 *
 * Insert a redirecting key, e.g. a KSK@ or SSK@ to a specific CHK@.
 * The original key is given in 'uri', perhaps in loose format w/o
 * the protocol specifier "entropy:".
 *
 * The key's fully qualified URI is stored in **fquri, which the caller
 * must xfree() at some point. The target of the redirect is given in 'chk'
 * and the resulting key - also a CHK - is returned in 'key'.
 *
 * The redirection is just disguised, not encrypted in a tamper proof
 * way, because the only thing we have is the fully qualified name of
 * the KSK@ or SSK@ to derive an encryption key from.
 * This name (like entropy:KSK@gpl.txt) is - or may be - known beforehand,
 * and so anyone can know the encryption (disguise) key.
 *
 * In other words: KSK@ and SSK@ can be hijacked. A "well known" SSK@ is by
 * no means a definite indication of the "same source" or "same inserter"
 * into the network, as anyone can create his own redirections in a known
 * name space. In fact the KSK@ based news system is showing the effect
 * all the time, because different content is inserted under the same
 * name. It's a Freenet legacy, so don't blame it on me.
 *
 * If you want signed data, you will have to use external tools such as
 * GnuPG to sign content spread through Entropy.
 *
 * @param key pointer to a chkey_t to receive the resulting CHK@ of the redirect
 * @param chk pointer to a chkey_t containing the target of the redirection
 * @param uri loose format uri of the key that shall redirect to the target
 * @param fquri pointer to a pointer to a string to receive the fully qualified uri
 * @param htl hops-to-live for the insert
 * @param collision pointer to an int that is set to 1 if the key was known
 * @param flags flags from enum with FILE_XXX constants
 *
 * @result zero on success, -1 and errno set on error
 */
int file_put_redirect(chkey_t *key, const chkey_t *chk, const char *uri,
	char **fquri, int htl, int *collision, int flags)
{
	char *xml = NULL;
	uint8_t *gzbuff = NULL;
	uint8_t *buff = NULL;
	size_t xmlsize, gzsize, size;
	int64_t limit, delay;
	int insflags, try, h;
	int e, rc = 0;
	FUN("file_put_redirect");

	*fquri = normalize_uri(uri);

	/* create the key from the fully qualified uri */
	if (0 != (rc = create_chk_from_ksk(key, *fquri))) {
		LOGS(L_FILE,L_ERROR,("ceate_chk_from_ksk('%s') call failed!\n",
			*fquri));
		goto bailout;
	}

	/* create a redirect XML body for that key */
	xmlsize = pm_asprintf(&xml, 
		"<?xml version=\"1.0\" standalone='yes'?>\n" \
		"<redirect key='%s' target='%s' />\n",
		key_long(key),
		key_long(chk));

	LOGS(L_FILE,L_DEBUG,("XML size: %#x ******\n%s",
		(unsigned)xmlsize, xml));

	LOGS(L_FILE,L_DEBUG,("redirect\n from: %s\n   to: %s\n",
		key_long(key),
		key_long(chk)));

	/* make the EK5 digest from the SHA1 digest */
	ek5_from_sha1(key->sha1.digest, &key->ek5);

	rc = file_squeeze(&gzbuff, &gzsize, xml, xmlsize);
	if (0 != rc) {
		LOGS(L_FILE,L_ERROR,("file_squeeze() call failed (%s)\n",
			strerror(errno)));
		goto bailout;
	}
	xfree(xml);

	if (gzsize <= 0 || gzsize >= CHUNKSIZE) {
		LOGS(L_FILE,L_DEBUGX,("Houston, we have a problem! gzsize %#x\n",
			(unsigned)gzsize));
		errno = EINVAL;
		rc = -1;
		goto bailout;
	}

	/* copy the compressed data */
	buff = xcalloc(CHUNKSIZE, sizeof(uint8_t));
	memcpy(buff, gzbuff, gzsize);
	xfree(gzbuff);
	size = gzsize;

#if	PARANOID == 0
	/* log the compressed buffer */
	LOGS(L_FILE,L_DEBUG,("compressed size: %#x\n%s\n",
		(unsigned)size, hexdump(buff, size)));
#endif

	/* encrypt with this EK5 hash - this actually only hides the redirect */
	key_crypt(key->ek5.digest, buff, size, KEY_CRYPT_GZ);

#if	PARANOID == 0
	LOGS(L_FILE,L_DEBUG,("encrypted size: %#x\n%s\n",
		(unsigned)size, hexdump(buff, size)));
#endif

	/* assume this is a new chunk */
	*collision = 0;
	if (htl >= 0) {
		if (0 == (rc = store_get(&key->sha1, NULL))) {
			LOGS(L_FILE,L_DEBUG,("key collision %s\n",
				sha1_hexshort(&key->sha1)));
			*collision = 1;
		}
		if (0 != (rc = store_put(&key->sha1, buff))) {
			LOGS(L_FILE,L_ERROR,("store_put(%s) call failed (%s)\n",
				sha1_hexshort(&key->sha1), strerror(errno)));
			goto bailout;
		}
	}

	LOGS(L_FILE,L_DEBUG,("URI %s, CHK %s, collision %d\n",
		*fquri, key_str(key), *collision));

	if (htl > 0) {
		insflags = 0;
		for (try = 1; try <= g_conf->fcpretries; try++) {
			h = retry_htl(htl, try);
			if (try + 1 < g_conf->fcpretries) {
				limit = MAX(g_peer->out.available, g_conf->bwlimit_out/32);
				delay = (uint64_t)1000000 * try *
					(MIN_MSGSIZE + size) / limit;
				/* No delays longer than MAXDELAY microseconds */
				delay = MIN(delay, MAXDELAY);
				LOGS(L_FILE,L_MINOR,("%s try #%d size:%#x HTL:%d %s\n",
					sha1_hexshort(&key->sha1), try,
					(unsigned)size, h, usec_str(delay)));
				osd_usleep(delay);
			}
			/* advertize the key to some peers if possible */
			rc = peer_ins_wait(&key->sha1, 1, &insflags, h, buff);
			if (0 == rc)
				break;
			if (EINVAL == errno)
				goto bailout;
		}
	}

	e = errno;
	if (0 != (flags & FILE_DEL)) {
		file_del_keys(&key->sha1, 1);
	}
	errno = e;

bailout:
	e = errno;
	xfree(buff);
	xfree(gzbuff);
	xfree(xml);
	errno = e;
	return rc;
}

/**
 * @brief Structure to keep the info of one fragment of a fragmented key
 */
typedef struct frag_s {
	chkey_t key;				/* fragment key */
	int mode;					/* mode (CODEC_FEC_8_4 ...) */
	size_t size;				/* size of this fragment (FRAGSIZE or less) */
	uint32_t n;				/* current chunk number */
	sha1_digest_t chunk[12];	/* chunk hash */
	uint8_t seen[12];			/* number of times we saw a hash= for a chunk */
}	frag_t;

/**
 * @brief Structure to keep the info of a fragmented key and all its fragments
 */
typedef struct content_s {
	size_t size;			/* size of the content */
	chkey_t key;			/* content key (short key) */
	int redir_seen;			/* seen a redirect */
	chkey_t redir;			/* redirect key */
	uint8_t *data;			/* possibly attached data */
	size_t nibble;			/* MSB/LSB toggle */
	size_t dataoffs;		/* offset into attached data */
	size_t datasize;		/* size of attached data */
	char *meta;				/* possibly attached metadata */
	size_t metaoffs;		/* offset into attached metadata */
	size_t metasize;		/* size of attached metadata */
	size_t fragcnt;			/* number of fragments */
	frag_t **frags;			/* list of fragments */
	frag_t *frag;			/* current fragment */
}	content_t;

/**
 * @brief XML parser callback for a <redirect ...> tag
 *
 * @param user user data (pointer to a content_t structure)
 * @param atts pointer to a list of attribute/value pairs
 */
void xml_start_redirect(void *user, const char **atts)
{
	content_t *c = user;
	const char **att;
	FUN("xml_start_redirect");

	c->redir_seen = 1;
	for (att = atts; *att; att += 2) {
		if (0 == strcasecmp(att[0], "target")) {
			if (0 != str_key(&c->redir, att[1])) {
				LOGS(L_FILE,L_ERROR,("invalid <redirect target=%s>\n", att[1]));
				continue;
			}
			LOGS(L_FILE,L_DEBUG,("<redirect target=%s>\n", key_str(&c->redir)));
		} else if (0 == strcasecmp(att[0], "key")) {
			if (0 != str_key(&c->key, att[1])) {
				LOGS(L_FILE,L_ERROR,("invalid <redirect key=%s>\n", att[1]));
				continue;
			}
			LOGS(L_FILE,L_DEBUG,("<redirect key=%s>\n", key_str(&c->key)));
		} else {
			LOGS(L_FILE,L_MINOR,("unhandled <redirect> attribute: %s='%s'\n",
				att[0], att[1]));
		}
	}
}

/**
 * @brief XML parser callback for a <content ...> tag
 *
 * @param user user data (pointer to a content_t structure)
 * @param atts pointer to a list of attribute/value pairs
 */
void xml_start_content(void *user, const char **atts)
{
	content_t *c = user;
	const char **att;
	uint32_t i, n;
	FUN("xml_start_content");

	for (att = atts; *att; att += 2) {
		if (0 == strcasecmp(att[0], "size")) {
			c->size = strtoul(att[1], NULL, 16);
			LOGS(L_FILE,L_DEBUG,("<content size=%x>\n",
				(unsigned)c->size));
		} else if (0 == strcasecmp(att[0], "key")) {
			if (0 != is_valid_bit(&c->key, att[1])) {
				LOGS(L_FILE,L_ERROR,("invalid <content key %s>\n", att[1]));
				continue;
			}
			LOGS(L_FILE,L_DEBUGX,("<content key=%s>\n", key_str(&c->key)));
		} else if (0 == strcasecmp(att[0], "fragments")) {
			c->fragcnt = n = strtoul(att[1], NULL, 16);
			if (n > 0) {
				c->frags = (frag_t **)xcalloc(n, sizeof(frag_t *));
				for (i = 0; i < n; i++) {
					c->frags[i] = (frag_t *)xcalloc(1, sizeof(frag_t));
				}
				LOGS(L_FILE,L_DEBUG,("allocated space for %u fragments\n",
					(unsigned)c->fragcnt));
			}
		} else {
			LOGS(L_FILE,L_ERROR,("unhandled <content> attribute: %s='%s'\n",
				att[0], att[1]));
		}
	}
}

/**
 * @brief XML parser callback for a <fragment ...> tag
 *
 * @param user user data (pointer to a content_t structure)
 * @param atts pointer to a list of attribute/value pairs
 */
void xml_start_fragment(void *user, const char **atts)
{
	content_t *c = user;
	const char **att;
	int mode = CODEC_UNKNOWN;
	FUN("xml_start_fragment");

	for (att = atts; *att; att += 2) {
		if (0 == strcasecmp(att[0], "number")) {
			uint32_t n = strtoul(att[1], NULL, 16);
			if (n >= c->fragcnt) {
				LOGS(L_FILE,L_ERROR,("fragment number too high (%u > %u)\n",
					(unsigned)n, (unsigned)c->fragcnt));
				c->frag = NULL;
			} else {
				c->frag = c->frags[n];
				if (CODEC_UNKNOWN != mode) {
					c->frag->mode = mode;
				}
			}
			LOGS(L_FILE,L_DEBUG,("frag #%#x at %p\n",
				(unsigned)(n + 1), c->frag));
		} else if (0 == strcasecmp(att[0], "mode")) {
			mode = CODEC_UNKNOWN;
			if (0 == strcmp(att[1], "fec84")) {
				mode = CODEC_FEC_8_4;
			}
			if (NULL != c->frag) {
				c->frag->mode = mode;
			}
		} else if (0 == strcasecmp(att[0], "size")) {
			if (NULL != c->frag) {
				c->frag->size = strtoul(att[1], NULL, 16);
			}
		} else if (0 == strcasecmp(att[0], "key")) {
			if (NULL != c->frag) {
				if (0 != is_valid_bit(&c->frag->key, att[1])) {
					LOGS(L_FILE,L_ERROR,("invalid <frag key %s>\n", att[1]));
					continue;
				}
				LOGS(L_FILE,L_DEBUG,("<frag key=%s>\n", key_str(&c->frag->key)));
			}
		} else {
			LOGS(L_FILE,L_ERROR,("unhandled <frag> attribute: %s='%s'\n",
				att[0], att[1]));
		}
	}
}

/**
 * @brief XML parser callback for a <chunk ...> tag
 *
 * @param user user data (pointer to a content_t structure)
 * @param atts pointer to a list of attribute/value pairs
 */
void xml_start_chunk(void *user, const char **atts)
{
	content_t *c = user;
	frag_t *f = c->frag;
	const char **att;
	FUN("xml_start_chunk");

	for (att = atts; *att; att += 2) {
		if (0 == strcasecmp(att[0], "number")) {
			f->n = strtoul(att[1], NULL, 16);
			if (f->n >= 12) {
				LOGS(L_FILE,L_ERROR,("invalid <chunk number=%x>\n",
					(unsigned)f->n));
			}
			LOGS(L_FILE,L_DEBUG,("chunk #%#x at %p\n",
				(unsigned)f->n, &f->chunk[f->n]));
		} else if (0 == strcasecmp(att[0], "hash")) {
			if (0 != sha1_strhex(&f->chunk[f->n], att[1])) {
				LOGS(L_FILE,L_ERROR,("invalid <chunk number=%x hash=%s>\n",
					f->n, att[1]));
				continue;
			}
			LOGS(L_FILE,L_DEBUG,("<chunk number=%x hash=%s />\n",
				(unsigned)f->n, sha1_hexshort(&f->chunk[f->n])));
			f->seen[f->n] += 1;
		} else {
			LOGS(L_FILE,L_ERROR,("unhandled <chunk> attribute: %s='%s'\n",
				att[0], att[1]));
		}
	}
}

/**
 * @brief XML parser callback for a <data ...> tag
 *
 * @param user user data (pointer to a content_t structure)
 * @param atts pointer to a list of attribute/value pairs
 */
void xml_start_data(void *user, const char **atts)
{
	content_t *c = user;
	const char **att;
	FUN("xml_start_data");

	xfree(c->meta);
	c->datasize = 0;

	for (att = atts; *att; att += 2) {
		if (0 == strcasecmp(att[0], "size")) {
			c->datasize = strtoul(att[1], NULL, 16);
			LOGS(L_FILE,L_DEBUG,("<data size=%x>\n",
				(unsigned)c->datasize));
		} else {
			LOGS(L_FILE,L_ERROR,("unhandled <data> attribute: %s='%s'\n",
				att[0], att[1]));
		}
	}

	if (c->datasize > 0) {
		c->data = (uint8_t *)xcalloc(c->datasize + 1, sizeof(char));
	}

	/* fill in the following CDATA */
	c->dataoffs = 0;
	c->nibble = 0;
}

/**
 * @brief XML parser callback for a <metadata ...> tag
 *
 * @param user user data (pointer to a content_t structure)
 * @param atts pointer to a list of attribute/value pairs
 */
void xml_start_metadata(void *user, const char **atts)
{
	content_t *c = user;
	const char **att;
	FUN("xml_start_metadata");

	xfree(c->meta);
	c->metasize = 0;

	for (att = atts; *att; att += 2) {
		if (0 == strcasecmp(att[0], "size")) {
			c->metasize = strtoul(att[1], NULL, 16);
			LOGS(L_FILE,L_DEBUG,("<metadata size=%x>\n",
				(unsigned)c->metasize));
		} else {
			LOGS(L_FILE,L_ERROR,("unhandled <metadata> attribute: %s='%s'\n",
				att[0], att[1]));
		}
	}

	if (c->metasize > 0) {
		c->meta = (char *)xcalloc(c->metasize + 1,1);
	}
	/* fill in the following CDATA */
	c->metaoffs = 0;
}

/**
 * @brief XML parser callback for the start of the key tags
 *
 * Dispatch to the callbacks for the various key tags.
 *
 * @param user user data (pointer to a content_t structure)
 * @param name pointer to a string with the name of the tag
 * @param atts pointer to a list of attribute/value pairs
 */
void xml_start_element(void *user, const char *name, const char **atts)
{
	FUN("xml_start_element");

	LOGS(L_FILE,L_DEBUG,("start: %s\n", name));
	if (0 == strcasecmp(name, "redirect")) {
		xml_start_redirect(user, atts);
	} else if (0 == strcasecmp(name, "content")) {
		xml_start_content(user, atts);
	} else if (0 == strcasecmp(name, "fragment")) {
		xml_start_fragment(user, atts);
	} else if (0 == strcasecmp(name, "chunk")) {
		xml_start_chunk(user, atts);
	} else if (0 == strcasecmp(name, "data")) {
		xml_start_data(user, atts);
	} else if (0 == strcasecmp(name, "metadata")) {
		xml_start_metadata(user, atts);
	} else {
		LOGS(L_FILE,L_ERROR,("unhandled start tag: <%s>\n", name));
	}
}

/**
 * @brief XML parser callback for the end of the key tags
 *
 * At the </metadata> tag verify if the string length matches
 * the attribute. If it does not, free the meta data and
 * reset the size.
 *
 * @param user user data (pointer to a content_t structure)
 * @param name pointer to a string with the name of the tag
 */
void xml_end_element(void *user, const char *name)
{
	content_t *c = user;
	FUN("xml_end_element");
	if (0 == strcasecmp(name, "metadata")) {
		size_t size = NULL != c->meta ?
			strlen(c->meta) : 0;
		if (size != c->metasize) {
			LOGS(L_FILE,L_ERROR,("metadata size mismatch:\n" \
				" got: %#x\n xml: %#x\n",
				(unsigned)size, (unsigned)c->metasize));
			xfree(c->meta);
			c->metasize = 0;
		} else if (size > 0) {
			LOGS(L_FILE,L_DEBUG,("metadata (%#x):\n%s",
				(unsigned)size, c->meta));
		}
	}
	LOGS(L_FILE,L_DEBUG,("end: %s\n", name));
}

static __inline uint8_t xdigit(char ch)
{
	if (ch >= '0' && ch <= '9')
		return (uint8_t)(ch - '0');
	if (ch >= 'A' && ch <= 'F')
		return (uint8_t)(ch - 'A' + 10);
	if (ch >= 'a' && ch <= 'f')
		return (uint8_t)(ch - 'a' + 10);
	return 0xff;
}

/**
 * @brief XML parser callback for the CDATA of <data ...> and <metadata ...> tags
 *
 * Decode the hexadecimal string in the CDATA back to binary data.
 * This code may be called back with arbitrary length hunks of the
 * CDATA, so we have to do each nibble separately.
 *
 * @param user user data (pointer to a content_t structure)
 * @param s pointer to a string with CDATA
 * @param len length of CDATA
 */
void xml_cdata(void *user, const char *s, int len)
{
	content_t *c = user;
	FUN("xml_cdata");
	if (c->datasize > 0 && c->dataoffs < c->datasize) {
		size_t size = 2 * (c->datasize - c->dataoffs);
		if (size > (size_t)len)
			size = (size_t)len;
		while (size > 0) {
			uint8_t ch = xdigit(*s++);
			if (0 == c->nibble) {
				c->data[c->dataoffs] = ch * 16;
				c->nibble = 1;
			} else {
				c->data[c->dataoffs] |= ch;
				c->nibble = 0;
				c->dataoffs += 1;
			}
			size--;
		}
	} else if (c->metasize > 0 && c->metaoffs < c->metasize) {
		size_t size = 2 * (c->metasize - c->metaoffs);
		if (size > (size_t)len)
			size = (size_t)len;
		while (size > 0) {
			uint8_t ch = xdigit(*s++);
			if (0 == c->nibble) {
				c->meta[c->metaoffs] = ch * 16;
				c->nibble = 1;
			} else {
				c->meta[c->metaoffs] |= ch;
				c->nibble = 0;
				c->metaoffs += 1;
			}
			size--;
		}
	}
}

/**
 * @brief XML parser callback for the uninteresting tabs, blanks, newlines etc.
 *
 * @param user user data (pointer to a content_t structure)
 * @param s pointer to a string with CDATA
 * @param len length of CDATA
 */
void xml_default(void *user, const char *s, int len)
{
	FUN("xml_default");
	(void)user;
	(void)s;
	(void)len;
}

/**
 * @brief Free a content_t structure data, metadata and fragments and zero it
 *
 * @param c pointer to a content_t structure
 */
void xml_free(content_t *c)
{
	xfree(c->data);
	xfree(c->meta);
	if (c->frags) {
		size_t i;
		for (i = 0; i < c->fragcnt; i++)
			xfree(c->frags[i]);
		xfree(c->frags);
	}
	memset(c, 0, sizeof(*c));
}

/**
 * @brief Get a file from the data store and/or network
 *
 * @param key pointer to a chkey_t containing the key to retrieve
 * @param tfdata pointer to a tempfile_t to receive the data
 * @param tfmeta pointer to a tempfile_t to receive the meta data
 * @param htl hops-to-live for the requests
 * @param slimit pointer to a size_t that (optionally) defines an upper size limit
 * @param user user data for callback
 * @param cb callback to progress info function
 * @param flags flags from enum with FILE_XXX constants
 *
 * @result zero on success, -1 and errno set on error
 */
int file_get(chkey_t *key, tempfile_t *tfdata, tempfile_t *tfmeta, int htl,
	size_t *slimit, void *user, file_callback cb, int flags)
{
	sha1_state_t ss;
	ek5_state_t es;
	chkey_t key_data, key_comp, key_frag;
	sha1_digest_t sha1_fec[1 + 12];
	uint8_t *buff = NULL, *code = NULL;
	size_t fragnum;
	char *xml = NULL;
	size_t xmlsize =  0;
	uint8_t *gzbuff = NULL;
	size_t copy = 0, gzoffs = 0, gzsize = 0;
	size_t totalsize = 0, datasize = 0, metasize = 0;
	size_t fragsize, n;
	int64_t limit, delay;
	int reqflags, advflags, bit, sum, try, h;
	XML_Parser par = NULL;
	content_t content;
	uint32_t fec_errors = 0;
	uint32_t frag_success = 0;
	uint32_t frag_errors = 0;
	uint32_t frag_failures = 0;
	uint32_t key_errors = 0;
	uint32_t key_failures = 0;
	int private = 0;
	int e = 0;
	int rc = 0;
	FUN("file_get");

	buff = (uint8_t *)xmalloc(FRAGSIZE);
	code = (uint8_t *)xmalloc(12 * CHUNKSIZE);
	memset(&content, 0, sizeof(content));

	if (NULL != tfdata) {
		if (0 != (rc = temp_open(tfdata))) {
			LOGS(L_FILE,L_ERROR,("temp_open('%s') call failed (%s)\n",
				tfdata->name ? tfdata->name : "[null]", strerror(errno)));
			goto bailout;
		}
	}

	if (NULL != tfmeta) {
		if (0 != (rc = temp_open(tfmeta))) {
			LOGS(L_FILE,L_ERROR,("temp_open('%s') call failed (%s)\n",
				tfmeta->name ? tfmeta->name : "[null]", strerror(errno)));
			goto bailout;
		}
	}

	key_data = *key;

	/* repeat until a non K_BIT key is found */
	for (;;) {

		xml_free(&content);

		if (gzsize > 0) {

			if (gzoffs < gzsize) {
				LOGS(L_FILE,L_ERROR,("incomplete %#x/%#x key %s\n",
					(unsigned)gzoffs, (unsigned)gzsize,
					sha1_hexshort(&key_data.sha1)));

				/* set xmlsize to 0, cannot expand the incomplete gzbuff */
				xmlsize = 0;
				goto bailout;

			} else {

#if	PARANOID == 0
				LOGS(L_FILE,L_DEBUGX,("size %#x/%#x key: %s\n%s\n",
					(unsigned)gzoffs, (unsigned)gzsize,
					sha1_hexshort(&key_data.sha1),
					hexdump(gzbuff, gzsize)));
#endif

				rc = file_expand(&xml, &xmlsize, gzbuff, gzsize);
				if (0 != rc) {
					e = errno;
					LOGS(L_FILE,L_ERROR,("standard key %s file_expand(%#x) failed (%s)\n",
						sha1_hexshort(&key_data.sha1),
						(unsigned)gzsize,
						strerror(errno)));
					store_zap(&key_data.sha1);
					frag_errors += 1;
					errno = e;
					goto bailout;
				}
#if	PARANOID == 0
				LOGS(L_FILE,L_DEBUGX,("expanded %#x\n%s\n",
					xmlsize, hexdump(xml, xmlsize)));
#endif

			}
			xfree(gzbuff);

		} else {

			/* zap document before requesting it */
			if (0 != (flags & FILE_ZAP)) {
				file_zap_keys(&key_data.sha1, 1);
			}

			reqflags = 0;
			memset(buff, 0, CHUNKSIZE);
			for (try = 1; try <= g_conf->fcpretries; try++) {
				h = retry_htl(htl, try);
				if (try > 1) {
					limit = MAX(g_peer->in.available, g_conf->bwlimit_in/32);
					delay = (int64_t)1000000 * try * MAX_MSGSIZE /
						limit / g_conf->fcpretries;
					/* No delays longer than MAXDELAY microseconds */
					delay = MIN(delay, MAXDELAY);
					LOGS(L_FILE,L_MINOR,("%s try #%d HTL:%d %s\n",
						sha1_hexshort(&key_data.sha1), try, h,
						usec_str(delay)));
					osd_usleep(delay);
				}
				rc = peer_req_wait(&key_data.sha1, 1, &reqflags, h, buff);
				if (0 == rc)
					break;
				if (EINVAL == errno)
					goto bailout;
				if (0 == htl)
					break;
			}

			if (0 != rc) {
				e = errno;
				LOGS(L_FILE,L_DEBUG,("peer_req_wait(%s) failed (%s)\n",
					key_short(&key_data), strerror(errno)));
				key_failures += 1;
				errno = e;
				goto bailout;
			}

			/* delete document after requesting it */
			if (0 != (flags & FILE_DEL)) {
				file_del_keys(&key_data.sha1, 1);
			}

			/* get the header size for this key */
			gzsize = ntohl(*(uint32_t*)&buff[4]);

			/* too small? */
			if (gzsize < 4 + 4 + 4) {
				LOGS(L_FILE,L_ERROR,("too small\n size:%#x\n key: %s\n",
					(unsigned)gzsize,
					sha1_hexshort(&key_data.sha1)));
				key_errors += 1;
				goto bailout;
			}

			/* too big? */
			if (gzsize >= CHUNKSIZE) {
				LOGS(L_FILE,L_ERROR,("too big\n size:%#x\n key: %s\n",
					(unsigned)gzsize,
					sha1_hexshort(&key_data.sha1)));
				key_errors += 1;
				goto bailout;
			}

#if	PARANOID == 0
			LOGS(L_FILE,L_DEBUGX,("key okay\n size:%#x\n key: %s\nencrypted:\n%s\n",
				(unsigned)gzsize,
				sha1_hexshort(&key_data.sha1),
				hexdump(buff, gzsize)));
#endif

			/* compare the SHA1 of the buffer with the key's SHA1 */
			sha1(buff, gzsize, &key_comp.sha1);

			if (0 == memcmp(&key_data.sha1, &key_comp.sha1, SHA1SIZE)) {
				LOGS(L_FILE,L_DEBUG,("SHA1 match - short key\n"));

				/* decrypt short key with the key_data EK5 */
				key_crypt(key_data.ek5.digest, buff, gzsize, KEY_CRYPT_GZ);
#if	PARANOID == 0
				LOGS(L_FILE,L_DEBUGX,("size:%#x key: %s\n" \
					"short key decrypted:\n%s\n",
					(unsigned)gzsize,
					sha1_hexshort(&key_data.sha1),
					hexdump(buff, gzsize)));
#endif

				rc = file_expand(&xml, &xmlsize, buff, gzsize);
				if (0 != rc) {
					e = errno;
					LOGS(L_FILE,L_ERROR,("short key %s file_expand(%#x) failed (%s)\n",
						sha1_hexshort(&key_data.sha1),
						(unsigned)gzsize,
						strerror(errno)));
					store_zap(&key_data.sha1);
					key_errors += 1;
					errno = e;
					goto bailout;
				}

			} else {

				LOGS(L_FILE,L_DEBUG,("SHA1 mismatch - redirect key\n"));

				/* make the EK5 digest from the SHA1 digest */
				ek5_from_sha1(key_data.sha1.digest, &key_data.ek5);

				/* decrypt redirecting key with this EK5 digest */
				key_crypt(key_data.ek5.digest, buff, gzsize, KEY_CRYPT_GZ);
#if	PARANOID == 0
				LOGS(L_FILE,L_DEBUGX,("size:%#x key: %s\n" \
					"redirect decrypted:\n%s\n",
					(unsigned)gzsize,
					sha1_hexshort(&key_data.sha1),
					hexdump(buff, gzsize)));
#endif

				rc = file_expand(&xml, &xmlsize, buff, gzsize);
				if (0 != rc) {
					e = errno;
					LOGS(L_FILE,L_ERROR,("redirect key %s file_expand(%#x) call failed (%s)\n",
						sha1_hexshort(&key->sha1),
						(unsigned)gzsize,
						strerror(errno)));
					store_zap(&key_data.sha1);
					store_zap(&key->sha1);
					key_errors += 1;
					errno = e;
					goto bailout;
				}
			}

#if	PARANOID == 0
			LOGS(L_FILE,L_DEBUGX,("key okay\n size:%#x\n key: %s\nexpanded:\n%s\n",
				(unsigned)xmlsize,
				sha1_hexshort(&key_data.sha1),
				hexdump(xml, xmlsize)));
#endif
		}

		gzoffs = 0;
		gzsize = 0;
		xml_free(&content);
		if (0 == xmlsize) {
			LOGS(L_FILE,L_ERROR,("XML buffer size for key %s is zero\n gzoffs: %#x\n gzsize: %#x\n",
				key_long(key), (unsigned)gzoffs, (unsigned)gzsize));
			errno = EINVAL;
			key_errors += 1;
			goto bailout;
		}
		LOGS(L_FILE,L_DEBUG,("XML parsing %u bytes for key %s\n%s",
			(unsigned)xmlsize, key_short(key), xml));
		if (NULL != par) {
			XML_ParserFree(par);
			par = NULL;
		}
		if (NULL == (par = XML_ParserCreate(NULL))) {
			LOGS(L_FILE,L_ERROR,("XML_ParserCreate() call failed (%s)\n",
				strerror(errno)));
			errno = ENOMEM;
			rc = -1;
			goto bailout;
		}
		XML_SetElementHandler(par,
			(XML_StartElementHandler)xml_start_element,
			(XML_EndElementHandler)xml_end_element);
		XML_SetCharacterDataHandler(par,
			(XML_CharacterDataHandler)xml_cdata);
		XML_SetDefaultHandler(par,
			(XML_DefaultHandler)xml_default);
		XML_SetUserData(par,
			(void *)&content);

		xml[xmlsize] = '\0';
#if	PARANOID == 0
		LOGS(L_FILE,L_DEBUG,("XML: size %u *****************\n%s\n",
			(unsigned)xmlsize, xml));
#endif

		/* **** Note: XML_Parse returns zero on error **** */
		if (0 == XML_Parse(par, xml, xmlsize, 1)) {
			e = errno;
			rc = XML_GetErrorCode(par);
			LOGS(L_FILE,L_ERROR,("XML_Parse returned %d (line %d, pos %d)\n",
				rc, XML_GetCurrentLineNumber(par),
					XML_GetCurrentColumnNumber(par)));
			store_zap(&key_data.sha1);
			key_errors += 1;
			rc = -1;
			errno = e;
			goto bailout;
		}

		xfree(xml);

		if (0 != content.redir_seen) {
			/* this is a redirect... */
			if (0 != memcmp(&key->sha1, &content.key.sha1, SHA1SIZE)) {
				LOGS(L_FILE,L_ERROR,("invalid redirect SHA1:%s for key:%s\n",
					sha1_hexshort(&content.key.sha1),
					sha1_hexshort(&key_data.sha1)));
				store_zap(&key_data.sha1);
				errno = EINVAL;
				rc = -1;
				goto bailout;
			}
			if (0 != memcmp(&key->ek5, &content.key.ek5, EK5SIZE)) {
				LOGS(L_FILE,L_ERROR,("invalid redirect EK5:%s for key:%s\n",
					ek5_hexshort(&content.key.ek5),
					ek5_hexshort(&key_data.ek5)));
				store_zap(&key_data.sha1);
				errno = EINVAL;
				rc = -1;
				goto bailout;
			}
#if	PARANOID == 0
			LOGS(L_FILE,L_DEBUG,("redirecting key\n from: %s\n   to: %s\n",
				key_long(&content.key), key_long(&content.redir)));
#endif
			*key = content.key;
			key_data = content.redir;
			continue;
		}

		private = (content.key.type[0] == MSB(K_BIT) &&
			content.key.type[1] == LSB(K_BIT));

		totalsize = content.size + content.metasize;

		LOGS(L_FILE,L_MINOR,("%s: %s frags:%u data:%#x meta:%#x\n",
			key_short(&key_data), (char *)(private ? "bit" : "key"),
			(unsigned)content.fragcnt,
			(unsigned)content.size,
			(unsigned)content.metasize));
		if (0 == totalsize) {
			LOGS(L_FILE,L_ERROR,("BUG: zero key found (length 0) %s\n",
				sha1_hexshort(&key->sha1)));
			if (0 == (rc = store_zap(&key->sha1))) {
				LOGS(L_FILE,L_DEBUG,("store_zap(%s) succeeded\n",
					sha1_hexshort(&key->sha1)));
			} else {
				LOGS(L_FILE,L_ERROR,("store_zap(%s) failed (%s)\n",
					sha1_hexshort(&key->sha1), strerror(errno)));
			}
		}

		if (NULL != slimit && *slimit > 0 && content.size > *slimit) {
			LOGS(L_FILE,L_MINOR,("%s: %s data %#x > %#x; return EFBIG\n",
				key_short(&key_data),
				(char *)(private ? "bit" : "key"),
				(unsigned)content.size,
				(unsigned)*slimit));
			*slimit = content.size;
			errno = EFBIG;
			rc = -1;
			goto bailout;
		}

		/* now let the race for the chunks begin... */
		for (try = 1; try <= g_conf->fcpretries; try++) {

			fec_errors = 0;
			key_errors = 0;
			frag_errors = 0;
			frag_failures = 0;
			frag_success = 0;
			datasize = 0;
			metasize = 0;
			if (NULL != tfdata) {
				temp_seek(tfdata, 0, SEEK_SET);
			}
			if (NULL != tfmeta) {
				temp_seek(tfmeta, 0, SEEK_SET);
			}
			xfree(gzbuff);

			/* if this is private then allocate gzbuff to collect data */
			if (0 != private) {
				if (content.size <= 0) {
					LOGS(L_FILE,L_ERROR,("%s content.size <= 0 (%#x)\n",
						sha1_hexshort(&key_data.sha1),
						(unsigned)gzsize));
					key_errors += 1;
					errno = EINVAL;
					rc = -1;
					goto bailout;
				}
				gzoffs = 0;
				gzsize = content.size;
				gzbuff = (uint8_t *)xmalloc(gzsize);
			}
	
			/* initialize the whole document data key state */
			sha1_init(&ss);
			ek5_init(&es);

			if (content.datasize > 0) {
				sha1_append(&ss, content.data, content.datasize);
				ek5_append(&es, content.data, content.datasize);
				if (0 != private) {
					copy = gzoffs + content.datasize > gzsize ?
						gzsize - gzoffs : content.datasize;
					LOGS(L_FILE,L_MINOR,("private data %#x @%#x/%#x\n",
						(unsigned)copy, (unsigned)gzoffs, (unsigned)gzsize));
					if (NULL == gzbuff) {
						LOGS(L_FILE,L_ERROR,("gzbuff is NULL\n"));
					} else if ((ssize_t)gzoffs < 0 || gzoffs > gzsize) {
						LOGS(L_FILE,L_ERROR,("bad gzoffs (%#x, size:%#x)\n",
							(unsigned)gzoffs, (unsigned)gzsize));
						key_errors += 1;
						goto bailout;
					} else if (copy > 0) {
						/* append to the LZW squeezed XML buffer */
						memcpy(&gzbuff[gzoffs], content.data, copy);
						gzoffs += copy;
					}
				} else if (NULL == tfdata) {
					copy = content.datasize > content.size ?
						content.size : content.datasize;
					LOGS(L_FILE,L_MINOR,("ignore %#x/%#x data for key %s\n",
						(unsigned)copy, (unsigned)content.datasize,
						key_short(key)));
					datasize += copy;
				} else {
					copy = content.datasize > content.size ?
						content.size : content.datasize;
					/* write to the frontend file */
					rc = temp_write(tfdata, content.data, copy);
					if (0 != rc) {
						LOGS(L_FILE,L_ERROR,("temp_write(%s) failed (%s)\n",
							tfdata->name, strerror(errno)));
						goto bailout;
					}
					datasize += copy;
				}

				if (NULL != cb) {
					file_info_t info;
					memset(&info, 0, sizeof(info));
					info.type = INFO_TYPE_PROGRESS;
					info.internal = flags & FILE_INT;
					info.hash = content.key.sha1;
					info.offs = datasize;
					info.size = totalsize;
					if (0 != (rc = (*cb)(user, &info))) {
						LOGS(L_FILE,L_ERROR,("callback failed (%s)\n",
							strerror(errno)));
						goto bailout;
					}
				}
			}

			for (fragnum = 0; fragnum < content.fragcnt; fragnum++) {
				frag_t *f = content.frags[fragnum];

				if (NULL != cb) {
					file_info_t info;
					memset(&info, 0, sizeof(info));
					info.type = INFO_TYPE_PROGRESS;
					info.internal = flags & FILE_INT;
					info.hash = content.frags[fragnum]->key.sha1;
					info.offs = fragnum * FRAGSIZE;
					info.size = totalsize;
					if (0 != (rc = (*cb)(user, &info))) {
						LOGS(L_FILE,L_ERROR,("callback failed (%s)\n",
							strerror(errno)));
						goto bailout;
					}
				}

				/* see if we shall zap chunks before retrieving them */
				if (1 == try) {
					if (0 != (flags & FILE_ZAP)) {
						file_zap_keys(f->chunk, 12);
					}
					if (htl > 0) {
						/* prepare the 13 peer_req_fec() hashes */
						memcpy(&sha1_fec[0], &f->key.sha1, SHA1SIZE);
						memcpy(&sha1_fec[1], f->chunk, 12 * SHA1SIZE);
						rc = peer_req_fec(sha1_fec, 2, htl, NULL, 0);
						if (0 == rc) {
							LOGS(L_FILE,L_MINOR,("peer_req_fec() succeeded\n"));
						} else {
							LOGS(L_FILE,L_MINOR,("peer_req_fec() failed (%s)\n",
								strerror(errno)));
						}
					}
				}
				/* request the 12 chunks */
				reqflags = 0;
				memset(code, 0, 12 * CHUNKSIZE);
				h = retry_htl(htl, try);
				rc = peer_req_wait(f->chunk, 12, &reqflags, h, code);
				if (EINVAL == errno)
					goto bailout;
				sum = bitsum(reqflags);
				if (0 != rc && sum >= 8) {
					LOGS(L_FILE,L_MINOR,("%d/12 chunks but rc was %d (%s)\n",
						sum, rc, strerror(errno)));
					rc = 0;
				}
				if (0 != rc) {
					LOGS(L_FILE,L_MINOR,("per_req_wait(%s) call failed (%s)\n",
						sha1_hexshort(&f->key.sha1), flags_str(reqflags)));
					frag_failures += 1;
					continue;
				}

				/* double check the hashes of the chunks */
				for (n = 0; n < 12; n++) {
					uint8_t *bits = &code[n * CHUNKSIZE];
					chkey_t key_chunk;

					/* skip chunks that we didn't receive */
					if (0 == (reqflags & (1 << n)))
						continue;

					sha1(bits, CHUNKSIZE, &key_chunk.sha1);

					rc = memcmp(&key_chunk.sha1, &f->chunk[n], SHA1SIZE);
					if (0 != rc) {
						LOGS(L_FILE,L_NORMAL,("%s frag #%#x/%#x #%#x bad\n",
							"fec84",
							(unsigned)(fragnum + 1),
							(unsigned)content.fragcnt,
							(unsigned)n));
						if (0 == store_zap(&f->chunk[n])) {
							LOGS(L_FILE,L_DEBUG,("store_zap(%s) succeeded\n",
								sha1_hexshort(&f->chunk[n])));
						} else if (ENOENT != errno) {
							LOGS(L_FILE,L_ERROR,("store_zap(%s) failed (%s)\n",
								sha1_hexshort(&f->chunk[n]),
								strerror(errno)));
						}
						/* erase this chunk */
						memset(bits, 0, CHUNKSIZE);
						/* and mark it as erasure */
						reqflags &= ~(1 << n);
					}
				}

				LOGS(L_FILE,L_MINOR,("%s decode frag #%#x/%#x size %#x\n",
					"fec84",
					(unsigned)(fragnum + 1),
					(unsigned)content.fragcnt,
					(unsigned)f->size));

				rc = fec84_decode(buff, code, FRAGSIZE, reqflags);
				if (0 != rc) {
					LOGS(L_FILE,L_MINOR,("%s decode frag #%#x/%#x rc:%d\n",
						"fec84",
						(unsigned)(fragnum + 1),
						(unsigned)content.fragcnt,
						rc));
					frag_failures += 1;
					continue;
				}

				/* check the fragment hash */
				sha1(buff, FRAGSIZE, &key_frag.sha1);
				rc = memcmp(&key_frag.sha1, &f->key.sha1, SHA1SIZE);
				if (0 != rc) {
					LOGS(L_FILE,L_ERROR,("%s frag #%#x/%#x key mismatch\n" \
						" len: %#x/%#x, reqflags: %s\n" \
						" xml: %s\n" \
						" got: %s\n",
						"fec84",
						(unsigned)(fragnum + 1),
						(unsigned)content.fragcnt,
						(unsigned)f->size, FRAGSIZE,
						flags_str(reqflags),
						sha1_hexshort(&f->key.sha1),
						sha1_hexshort(&key_frag.sha1)));
					frag_failures += 1;
					continue;
				}
				frag_success += 1;

				LOGS(L_FILE,L_DEBUG,("frag #%#x SHA1 %s\n",
					(unsigned)(fragnum + 1),
					sha1_hexshort(&key_frag.sha1)));

				LOGS(L_FILE,L_MINOR,("%s frag #%#x/%#x key seems good (%s)\n",
					"fec84",
					(unsigned)(fragnum + 1), 
					(unsigned)content.fragcnt,
					flags_str(reqflags)));

				/*
				 * Encode the missing check chunks, except the ones that are
				 * non zero in the reqflags. fec84_encode() encodes chunks
				 * which have bit N for chunk #N (0 to 11) clear.
				 * Chunks #0 to #7 are always there after decoding.
				 */
				advflags = reqflags | 0x0ff;
				rc = fec84_encode(code, buff, FRAGSIZE, advflags);

				if (0 != rc) {
					LOGS(L_FILE,L_MINOR,("%s encode frag #%#x/%#x returned %d\n",
						"fec84",
						(unsigned)(fragnum + 1),
						(unsigned)content.fragcnt,
						rc));
					frag_failures += 1;
					continue;
				}

				/* append encrypted buff to the document SHA1 and EK5 state */
				sha1_append(&ss, buff, FRAGSIZE);
				ek5_append(&es, buff, FRAGSIZE);


				/* find the true fragment size */
				fragsize = f->size < FRAGSIZE ? f->size : FRAGSIZE;

				/* decrypt the fragment */
				key_crypt(f->key.ek5.digest, buff, fragsize, 0);

				/* check the EK5 hash of the decrypted fragment */
				ek5(buff, FRAGSIZE, &key_frag.ek5);
				rc = memcmp(&key_frag.ek5, &f->key.ek5, EK5SIZE);

				if (0 != rc) {
					LOGS(L_FILE,L_ERROR,("%s frag #%#x/%#x EK5 mismatch\n" \
						" len: %#x/%#x, reqflags: %s\n" \
						" xml: %s\n" \
						" got: %s\n",
						"fec84",
						(unsigned)(fragnum + 1),
						(unsigned)content.fragcnt,
						(unsigned)f->size, FRAGSIZE,
						flags_str(reqflags),
						ek5_hexshort(&f->key.ek5),
						ek5_hexshort(&key_frag.ek5)));
					frag_errors += 1;
					continue;
				}

				/* check all chunks and store the reconstructed ones */
				for (n = 0; n < 12; n++) {
					uint8_t *data = &code[n * CHUNKSIZE];
					chkey_t key_chunk;

					bit = 1 << n;

					sha1(data, CHUNKSIZE, &key_chunk.sha1);

					rc = memcmp(&key_chunk.sha1, &f->chunk[n], SHA1SIZE);
					if (0 != rc) {
						LOGS(L_FILE,L_NORMAL,("%s frag #%#x/%#x chunk #%#x bad\n",
							"fec84",
							(unsigned)(fragnum + 1),
							(unsigned)content.fragcnt,
							(unsigned)n));
						/* erase this chunk */
						memset(data, 0, CHUNKSIZE);
						frag_errors += 1;
						goto bailout;
					}
					/* skip chunks that we already had */
					if (0 != (reqflags & bit))
						continue;
					if (0 != (rc = store_put(&f->chunk[n], data))) {
						LOGS(L_FILE,L_ERROR,("store_put(%s) #%#x failed (%s)\n",
							sha1_hexshort(&f->chunk[n]), (unsigned)n,
							strerror(errno)));
						goto bailout;
					}
					LOGS(L_FILE,L_MINOR,("store_put(%s) #%#x succeeded\n",
						sha1_hexshort(&f->chunk[n]), (unsigned)n));
					/* and mark as available */
					reqflags |= bit;
				}

				if (1 == try && 0 != (flags & FILE_DEL)) {
					file_del_keys(f->chunk, 12);
				}

				if (htl > 0) {
					uint32_t bits;
					/* advertize up to 3 chunks */
					if (0 != rnd_get(&bits, sizeof(bits))) {
						const char *errmsg = strerror(errno);
						LOGS(L_FILE,L_ERROR,("rnd_get() failed (%s)\n",
							errmsg));
						die(1, "rnd_get() failed (%s)", errmsg);
					}
					/* set bits of advflags from 3 nibbles of 'bits' */
					advflags = 1 << ((bits >> 0) % 12);
					advflags |= 1 << ((bits >> 4) % 12);
					advflags |= 1 << ((bits  >> 8) % 12);
					LOGS(L_FILE,L_MINOR,("ADV randomly selected %s\n",
						flags_str(advflags)));
					for (n = 0; n < 12; n++) {
						bit = 1 << n;
						if (0 == (advflags & bit))
							continue;
						/* no error checking for timeouts etc. */
						peer_adv_key(&f->chunk[n], try, htl, NULL, 0);
					}
				}

				if (0 != private) {
					/* number of bytes to copy */
					copy = gzoffs + f->size > gzsize ?
						gzsize - gzoffs : f->size;
					LOGS(L_FILE,L_MINOR,("private data %#x @%#x/%#x\n",
						(unsigned)copy, (unsigned)gzoffs, (unsigned)gzsize));
					if (NULL == gzbuff) {
						LOGS(L_FILE,L_ERROR,("gzbuff is NULL\n"));
					} else if ((ssize_t)gzoffs < 0 || gzoffs > gzsize) {
						LOGS(L_FILE,L_ERROR,("gzoffs (%#x, size:%#x)\n",
							(unsigned)gzoffs, (unsigned)gzsize));
						key_errors += 1;
						goto bailout;
					} else if (copy > 0) {
						/* append to the LZW squeezed XML buffer */
						memcpy(&gzbuff[gzoffs], buff, copy);
						gzoffs += copy;
						datasize += copy;
					}
	
				} else if (NULL == tfdata) {
					/* number of bytes to ignore */
					copy = f->size > (datasize + content.size) ?
						(content.size - datasize) : f->size;
					LOGS(L_FILE,L_MINOR,("ignore %#x/%#x data for key %s\n",
						(unsigned)copy, (unsigned)f->size,
						sha1_hexshort(&key->sha1)));
					datasize += copy;
				} else {
					/* number of bytes to write */
					copy = f->size > (datasize + content.size) ?
						(content.size - datasize) : f->size;
					/* write to the frontend file */
					if (0 != (rc = temp_write(tfdata, buff, copy))) {
						LOGS(L_FILE,L_ERROR,("temp_write(%s) failed (%s)\n",
							tfdata->name, strerror(errno)));
						goto bailout;
					}
					datasize += copy;
				}

				/* break out of the loop if the content size is reached */
				if (datasize >= content.size)
					break;
			}

			if (content.metasize > 0) {
				copy = content.metasize;
				ek5_append(&es, content.meta, copy);
				if (NULL == tfmeta) {
					LOGS(L_FILE,L_DEBUG,("ignoring %#x meta for key %s\n",
						(unsigned)copy, key_short(key)));
					copy = 0;
				} else if (0 != (rc = temp_write(tfmeta, content.meta, copy))) {
					LOGS(L_FILE,L_ERROR,("temp_write(%s) %#x failed (%s)\n",
						tfmeta->name, (unsigned)copy, strerror(errno)));
					goto bailout;
				}
				metasize = copy;
			}

			sha1_finish(&ss, &key_comp.sha1);
			ek5_finish(&es, &key_comp.ek5);

			/* Bad key if:
		 	* a) SHA1 over all encrypted fragments does not match or
		 	* b) EK5 over all encrypted fragments + meta data does not match or
		 	* c) content.size is > 0, i.e. missing data
		 	*/
			if (datasize != content.size ||
				0 != memcmp(&key_comp.sha1, &content.key.sha1, SHA1SIZE) ||
				(0 == private &&
					0 != memcmp(&key_comp.ek5, &content.key.ek5, EK5SIZE))) {
				if (frag_errors > 0 || fec_errors > 0) {
					/* log an error only if datasize is greater than zero */
					if (datasize > 0) {
						LOGS(L_FILE,L_ERROR,(
							"bad fragment(s): %#x data, %#x meta\n" \
							"\t%u FEC errors, %u/%u/%u frags err/good/total\n" \
							"\tfor %s (%d)\n" \
							"\txml %s,%s\n" \
							"\tgot %s,%s\n",
							(unsigned)datasize,
							(unsigned)metasize,
							fec_errors, frag_errors, frag_success,
							(unsigned)content.fragcnt,
							sha1_hexshort(&key->sha1), key->log2size,
							sha1_hexshort(&content.key.sha1),
							ek5_hexshort(&content.key.ek5),
							sha1_hexshort(&key_comp.sha1),
							ek5_hexshort(&key_comp.ek5)));
					}
					errno = EINVAL;
					rc = -1;
					break;
				}
				/* log only if we got all data and still have a key error */
				if (datasize == content.size) {
					LOGS(L_FILE,L_ERROR,(
						"key mismatch; %#x data, %#x meta\n" \
						"\txml %s,%s\n" \
						"\tgot %s,%s\n",
						(unsigned)datasize,
						(unsigned)metasize,
						sha1_hexshort(&content.key.sha1),
						ek5_hexshort(&content.key.ek5),
						sha1_hexshort(&key_comp.sha1),
						ek5_hexshort(&key_comp.ek5)));
				}
				errno = EAGAIN;
				rc = -1;
				break;
			}

			if (0 != memcmp(&key_data.ek5, &content.key.ek5, EK5SIZE) &&
				0 == content.redir_seen) {
				LOGS(L_FILE,L_ERROR,("EK5 chaining error old:%s new:%s\n",
					ek5_hexshort(&key_data.ek5),
					ek5_hexshort(&content.key.ek5)));
			}

			if (htl > 0) {
				/* if everything went okay, we can now safely adv the key */
				peer_adv_key(&key_data.sha1, 2, htl, NULL, 0);
			}

			if (0 != rc && EINVAL == errno)
				break;

			if (0 == frag_failures && content.fragcnt == frag_success)
				break;

			limit = MAX(g_peer->in.available, g_conf->bwlimit_in / 32);
			delay = (uint64_t)1000000 * try * 12 * MAX_MSGSIZE /
				limit;
			/* No delays longer than MAXDELAY microseconds */
			delay = MIN(delay, MAXDELAY);
			LOGS(L_FILE,L_MINOR,("after try #%d sleep %s\n",
				try, usec_str(delay)));
			osd_usleep(delay);
		}

		/* break out if we have an EINVAL error */
		if (0 != rc && EINVAL == errno)
			break;

		/* climb down the XML tree, if any */
		key_data = content.key;

		/* If this is not a K_BIT key, we have the top level data */
		if (MSB(K_BIT) != key_data.type[0] || LSB(K_BIT) != key_data.type[1])
			break;
	}

bailout:
	e = errno;
	xml_free(&content);
	xfree(buff);
	xfree(code);
	xfree(gzbuff);
	if (NULL != par) {
		XML_ParserFree(par);
		par = NULL;
	}
	temp_close(tfdata, 0);
	temp_close(tfmeta, 0);
	if (frag_errors > 0) {
		LOGS(L_FILE,L_ERROR,("%s: %u fragment error(s)\n",
			key_short(key), fec_errors));
		if (0 == (rc = store_zap(&key->sha1))) {
			LOGS(L_FILE,L_DEBUG,("store_zap(%s) succeeded\n",
				sha1_hexshort(&key->sha1)));
		} else {
			LOGS(L_FILE,L_ERROR,("store_zap(%s) failed (%s)\n",
				sha1_hexshort(&key->sha1), strerror(errno)));
		}
		e = errno = EINVAL;
		rc = -1;
	} else if (fec_errors > 0) {
		LOGS(L_FILE,L_ERROR,("%s: %u FEC error(s)\n",
			key_short(key), fec_errors));
		if (0 == (rc = store_zap(&key->sha1))) {
			LOGS(L_FILE,L_DEBUG,("store_zap(%s) succeeded\n",
				sha1_hexshort(&key->sha1)));
		} else {
			LOGS(L_FILE,L_ERROR,("store_zap(%s) failed (%s)\n",
				sha1_hexshort(&key->sha1), strerror(errno)));
		}
		e = errno = EINVAL;
		rc = -1;
	} else if (key_errors > 0) {
		LOGS(L_FILE,L_DEBUG,("%s: %u key errors\n",
			key_short(key), key_errors));
		if (0 == (rc = store_zap(&key->sha1))) {
			LOGS(L_FILE,L_DEBUG,("store_zap(%s) succeeded\n",
				sha1_hexshort(&key->sha1)));
		} else {
			LOGS(L_FILE,L_ERROR,("store_zap(%s) failed (%s)\n",
				sha1_hexshort(&key->sha1), strerror(errno)));
		}
		e = errno = EAGAIN;
		rc = -1;
	} else if (key_failures > 0) {
		LOGS(L_FILE,L_DEBUG,("%s: %u key(s) failed\n",
			key_short(key), key_failures));
		e = errno = EAGAIN;
		rc = -1;
	} else if (frag_failures > 0) {
		LOGS(L_FILE,L_DEBUG,("%s: %u fragment(s) failed\n",
			key_short(key), frag_failures));
		e = errno = EAGAIN;
		rc = -1;
	} else if (0 == xmlsize || (0 == datasize && 0 == metasize)) {
		LOGS(L_FILE,L_ERROR,("%s: no data found\n",
			key_short(key)));
		e = errno = EAGAIN;
		rc = -1;
	} else {

		LOGS(L_FILE,L_MINOR,("%s: %#x data, %#x metadata\n",
			key_short(key), (unsigned)datasize, (unsigned)metasize));

		if (htl > 0) {
			/* if everything went okay, we can safely advertize this key */
			peer_adv_key(&key->sha1, 2, htl, NULL, 0);
		}

		if (NULL != cb) {
			file_info_t info;

			memset(&info, 0, sizeof(info));
			info.type = INFO_TYPE_COMPLETED;
			info.internal = flags & FILE_INT;
			info.hash = key->sha1;
			info.offs = datasize + metasize;
			info.size = datasize + metasize;
			if (0 != (*cb)(user, &info)) {
				LOGS(L_FILE,L_ERROR,("callback failed (%s)\n",
					strerror(errno)));
			}
		}
	}
	errno = e;
	return rc;
}
